{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:6IL5SNEOH3WB3JUDON72G4ZQOV","short_pith_number":"pith:6IL5SNEO","canonical_record":{"source":{"id":"2406.14598","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-06-20T17:56:07Z","cross_cats_sorted":[],"title_canon_sha256":"c2e73424dbc4f04619d46ab8005fedbe3e954404747798b3d1ad88a7d4b14513","abstract_canon_sha256":"7d4e293df6031a47a7b625cc0f99eae6537624dfcb1de49947fa06a3f5cf8fa4"},"schema_version":"1.0"},"canonical_sha256":"f217d9348e3eec1da683737fa37330757253dc2301ef01507740739934d801c8","source":{"kind":"arxiv","id":"2406.14598","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.14598","created_at":"2026-07-05T10:22:01Z"},{"alias_kind":"arxiv_version","alias_value":"2406.14598v2","created_at":"2026-07-05T10:22:01Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.14598","created_at":"2026-07-05T10:22:01Z"},{"alias_kind":"pith_short_12","alias_value":"6IL5SNEOH3WB","created_at":"2026-07-05T10:22:01Z"},{"alias_kind":"pith_short_16","alias_value":"6IL5SNEOH3WB3JUD","created_at":"2026-07-05T10:22:01Z"},{"alias_kind":"pith_short_8","alias_value":"6IL5SNEO","created_at":"2026-07-05T10:22:01Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:6IL5SNEOH3WB3JUDON72G4ZQOV","target":"record","payload":{"canonical_record":{"source":{"id":"2406.14598","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-06-20T17:56:07Z","cross_cats_sorted":[],"title_canon_sha256":"c2e73424dbc4f04619d46ab8005fedbe3e954404747798b3d1ad88a7d4b14513","abstract_canon_sha256":"7d4e293df6031a47a7b625cc0f99eae6537624dfcb1de49947fa06a3f5cf8fa4"},"schema_version":"1.0"},"canonical_sha256":"f217d9348e3eec1da683737fa37330757253dc2301ef01507740739934d801c8","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:22:01.456762Z","signature_b64":"EoJPkuWh9NjRCV7EU8s2lMiwR18c9xmNMxew2uSeXlTJtfw9qhbzM602ouK5lSESvksGWP6PtBKkykq1PlcECw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f217d9348e3eec1da683737fa37330757253dc2301ef01507740739934d801c8","last_reissued_at":"2026-07-05T10:22:01.456080Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:22:01.456080Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.14598","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:22:01Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"2HSst+2AKrEiGt8iCdNJnlIVbucPu/ZC3d5xUb21YZQtOSM1OC/pgM/f4CQQ+ZCNlNRCRsYJmxb8URFuIYbRDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-10T13:01:29.325017Z"},"content_sha256":"19a9514ac7d832fab6cbde8334bdf602a2e1fdf43dc802751eff802f7a93dd1e","schema_version":"1.0","event_id":"sha256:19a9514ac7d832fab6cbde8334bdf602a2e1fdf43dc802751eff802f7a93dd1e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:6IL5SNEOH3WB3JUDON72G4ZQOV","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"SORRY-Bench: Systematically Evaluating Large Language Model Safety Refusal","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Bo Li, Boyi Wei, Dacheng Li, Danqi Chen, Kai Li, Kaixuan Huang, Luxi He, Peter Henderson, Prateek Mittal, Ruoxi Jia, Tinghao Xie, Udari Madhushani Sehwag, Xiangyu Qi, Yangsibo Huang, Ying Sheng, Yi Zeng","submitted_at":"2024-06-20T17:56:07Z","abstract_excerpt":"Evaluating aligned large language models' (LLMs) ability to recognize and reject unsafe user requests is crucial for safe, policy-compliant deployments. Existing evaluation efforts, however, face three limitations that we address with SORRY-Bench, our proposed benchmark. First, existing methods often use coarse-grained taxonomies of unsafe topics, and are over-representing some fine-grained topics. For example, among the ten existing datasets that we evaluated, tests for refusals of self-harm instructions are over 3x less represented than tests for fraudulent activities. SORRY-Bench improves o"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.14598","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.14598/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:22:01Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"NoAkYGPx0UfZ0g0dtyYu4PVP3y9+lLqdrfwFHKw3tjFAcDVPu5/pPIEB1L7GoTLYdKPWCqKiL2Jtidr1EEcwDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-10T13:01:29.325417Z"},"content_sha256":"b3fdc223b75ac0e1c8369e1a36b0b02bdb1267a6aa12832ffb768a251a0a21da","schema_version":"1.0","event_id":"sha256:b3fdc223b75ac0e1c8369e1a36b0b02bdb1267a6aa12832ffb768a251a0a21da"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/6IL5SNEOH3WB3JUDON72G4ZQOV/bundle.json","state_url":"https://pith.science/pith/6IL5SNEOH3WB3JUDON72G4ZQOV/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/6IL5SNEOH3WB3JUDON72G4ZQOV/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-10T13:01:29Z","links":{"resolver":"https://pith.science/pith/6IL5SNEOH3WB3JUDON72G4ZQOV","bundle":"https://pith.science/pith/6IL5SNEOH3WB3JUDON72G4ZQOV/bundle.json","state":"https://pith.science/pith/6IL5SNEOH3WB3JUDON72G4ZQOV/state.json","well_known_bundle":"https://pith.science/.well-known/pith/6IL5SNEOH3WB3JUDON72G4ZQOV/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:6IL5SNEOH3WB3JUDON72G4ZQOV","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7d4e293df6031a47a7b625cc0f99eae6537624dfcb1de49947fa06a3f5cf8fa4","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-06-20T17:56:07Z","title_canon_sha256":"c2e73424dbc4f04619d46ab8005fedbe3e954404747798b3d1ad88a7d4b14513"},"schema_version":"1.0","source":{"id":"2406.14598","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.14598","created_at":"2026-07-05T10:22:01Z"},{"alias_kind":"arxiv_version","alias_value":"2406.14598v2","created_at":"2026-07-05T10:22:01Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.14598","created_at":"2026-07-05T10:22:01Z"},{"alias_kind":"pith_short_12","alias_value":"6IL5SNEOH3WB","created_at":"2026-07-05T10:22:01Z"},{"alias_kind":"pith_short_16","alias_value":"6IL5SNEOH3WB3JUD","created_at":"2026-07-05T10:22:01Z"},{"alias_kind":"pith_short_8","alias_value":"6IL5SNEO","created_at":"2026-07-05T10:22:01Z"}],"graph_snapshots":[{"event_id":"sha256:b3fdc223b75ac0e1c8369e1a36b0b02bdb1267a6aa12832ffb768a251a0a21da","target":"graph","created_at":"2026-07-05T10:22:01Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.14598/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Evaluating aligned large language models' (LLMs) ability to recognize and reject unsafe user requests is crucial for safe, policy-compliant deployments. Existing evaluation efforts, however, face three limitations that we address with SORRY-Bench, our proposed benchmark. First, existing methods often use coarse-grained taxonomies of unsafe topics, and are over-representing some fine-grained topics. For example, among the ten existing datasets that we evaluated, tests for refusals of self-harm instructions are over 3x less represented than tests for fraudulent activities. SORRY-Bench improves o","authors_text":"Bo Li, Boyi Wei, Dacheng Li, Danqi Chen, Kai Li, Kaixuan Huang, Luxi He, Peter Henderson, Prateek Mittal, Ruoxi Jia, Tinghao Xie, Udari Madhushani Sehwag, Xiangyu Qi, Yangsibo Huang, Ying Sheng, Yi Zeng","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-06-20T17:56:07Z","title":"SORRY-Bench: Systematically Evaluating Large Language Model Safety Refusal"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.14598","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:19a9514ac7d832fab6cbde8334bdf602a2e1fdf43dc802751eff802f7a93dd1e","target":"record","created_at":"2026-07-05T10:22:01Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7d4e293df6031a47a7b625cc0f99eae6537624dfcb1de49947fa06a3f5cf8fa4","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-06-20T17:56:07Z","title_canon_sha256":"c2e73424dbc4f04619d46ab8005fedbe3e954404747798b3d1ad88a7d4b14513"},"schema_version":"1.0","source":{"id":"2406.14598","kind":"arxiv","version":2}},"canonical_sha256":"f217d9348e3eec1da683737fa37330757253dc2301ef01507740739934d801c8","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f217d9348e3eec1da683737fa37330757253dc2301ef01507740739934d801c8","first_computed_at":"2026-07-05T10:22:01.456080Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:22:01.456080Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"EoJPkuWh9NjRCV7EU8s2lMiwR18c9xmNMxew2uSeXlTJtfw9qhbzM602ouK5lSESvksGWP6PtBKkykq1PlcECw==","signature_status":"signed_v1","signed_at":"2026-07-05T10:22:01.456762Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.14598","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:19a9514ac7d832fab6cbde8334bdf602a2e1fdf43dc802751eff802f7a93dd1e","sha256:b3fdc223b75ac0e1c8369e1a36b0b02bdb1267a6aa12832ffb768a251a0a21da"],"state_sha256":"83163d104a06b27fa1f0e09c3c87e22df29adc90b695b3c058ffdafa073550c4"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"GoHVWJ42LTpd09RPZ4fg0RrTYFcrbChtqVap9P174IV/nkAjWeNn1oTH/m+1atgvLwgH8rv5pMdTaTqHRh4JAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-10T13:01:29.327459Z","bundle_sha256":"55468b1ab343726dbec7128d24fbc5bc059b7863e0cd6dede7741b0d0bef6221"}}