{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:OD7T7KHGIJS2GD536GJITNJ7QM","short_pith_number":"pith:OD7T7KHG","canonical_record":{"source":{"id":"2507.09820","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-07-13T22:34:20Z","cross_cats_sorted":["cs.CY"],"title_canon_sha256":"007259881f5446f46d6028a10e8b3e1eabe662619d7215cb9d9f470d48e4a022","abstract_canon_sha256":"5657404bd91ba30897cdded25766eb115edf30f279186fe592c003d016e7b032"},"schema_version":"1.0"},"canonical_sha256":"70ff3fa8e64265a30fbbf19289b53f83232e929707b876510d4cc8fe6b8097df","source":{"kind":"arxiv","id":"2507.09820","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2507.09820","created_at":"2026-07-05T11:36:40Z"},{"alias_kind":"arxiv_version","alias_value":"2507.09820v1","created_at":"2026-07-05T11:36:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2507.09820","created_at":"2026-07-05T11:36:40Z"},{"alias_kind":"pith_short_12","alias_value":"OD7T7KHGIJS2","created_at":"2026-07-05T11:36:40Z"},{"alias_kind":"pith_short_16","alias_value":"OD7T7KHGIJS2GD53","created_at":"2026-07-05T11:36:40Z"},{"alias_kind":"pith_short_8","alias_value":"OD7T7KHG","created_at":"2026-07-05T11:36:40Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:OD7T7KHGIJS2GD536GJITNJ7QM","target":"record","payload":{"canonical_record":{"source":{"id":"2507.09820","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-07-13T22:34:20Z","cross_cats_sorted":["cs.CY"],"title_canon_sha256":"007259881f5446f46d6028a10e8b3e1eabe662619d7215cb9d9f470d48e4a022","abstract_canon_sha256":"5657404bd91ba30897cdded25766eb115edf30f279186fe592c003d016e7b032"},"schema_version":"1.0"},"canonical_sha256":"70ff3fa8e64265a30fbbf19289b53f83232e929707b876510d4cc8fe6b8097df","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:36:40.301941Z","signature_b64":"UEzdErXvIcf10y/ESpyO7uI3gtZpNogXOT8KYQcpXYnYjh7qJYpmZ2VDv79mO5WG1O1Unorp7w7wHqj2QIdoAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"70ff3fa8e64265a30fbbf19289b53f83232e929707b876510d4cc8fe6b8097df","last_reissued_at":"2026-07-05T11:36:40.301402Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:36:40.301402Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2507.09820","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:36:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ZF7kCo/CKL9RPS3+8U/Z/kDPBDVN4xa96pEXzm7En/o37lQsET1yKL5SFW+U58i0SOfi6dNllNVP6gC6KWf5BQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T12:39:01.199457Z"},"content_sha256":"cbd4ced571ff26c8463d53497d38721a3b2314fee783fdfc92aadd52acdda989","schema_version":"1.0","event_id":"sha256:cbd4ced571ff26c8463d53497d38721a3b2314fee783fdfc92aadd52acdda989"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:OD7T7KHGIJS2GD536GJITNJ7QM","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Measuring What Matters: A Framework for Evaluating Safety Risks in Real-World LLM Applications","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CY"],"primary_cat":"cs.SE","authors_text":"Gabriel Chua, Jessica Foo, Jia Yi Goh, Leanne Tan, Nyx Iskandar, Shaun Khoo","submitted_at":"2025-07-13T22:34:20Z","abstract_excerpt":"Most safety testing efforts for large language models (LLMs) today focus on evaluating foundation models. However, there is a growing need to evaluate safety at the application level, as components such as system prompts, retrieval pipelines, and guardrails introduce additional factors that significantly influence the overall safety of LLM applications. In this paper, we introduce a practical framework for evaluating application-level safety in LLM systems, validated through real-world deployment across multiple use cases within our organization. The framework consists of two parts: (1) princi"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2507.09820","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2507.09820/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:36:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"6DLCVyzvWT2hVgJQPDMYFLd7w7OqEWIZIz+hFwWebfDtRQn3RXuVuid4h3YauGiKOMA56DH7DmLsBljCKF1PCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T12:39:01.199830Z"},"content_sha256":"4427a62b656eb0d0f0c774acbf4c36bb962b79d7307bfeb00ba7711c9471c679","schema_version":"1.0","event_id":"sha256:4427a62b656eb0d0f0c774acbf4c36bb962b79d7307bfeb00ba7711c9471c679"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/OD7T7KHGIJS2GD536GJITNJ7QM/bundle.json","state_url":"https://pith.science/pith/OD7T7KHGIJS2GD536GJITNJ7QM/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/OD7T7KHGIJS2GD536GJITNJ7QM/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T12:39:01Z","links":{"resolver":"https://pith.science/pith/OD7T7KHGIJS2GD536GJITNJ7QM","bundle":"https://pith.science/pith/OD7T7KHGIJS2GD536GJITNJ7QM/bundle.json","state":"https://pith.science/pith/OD7T7KHGIJS2GD536GJITNJ7QM/state.json","well_known_bundle":"https://pith.science/.well-known/pith/OD7T7KHGIJS2GD536GJITNJ7QM/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:OD7T7KHGIJS2GD536GJITNJ7QM","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"5657404bd91ba30897cdded25766eb115edf30f279186fe592c003d016e7b032","cross_cats_sorted":["cs.CY"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-07-13T22:34:20Z","title_canon_sha256":"007259881f5446f46d6028a10e8b3e1eabe662619d7215cb9d9f470d48e4a022"},"schema_version":"1.0","source":{"id":"2507.09820","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2507.09820","created_at":"2026-07-05T11:36:40Z"},{"alias_kind":"arxiv_version","alias_value":"2507.09820v1","created_at":"2026-07-05T11:36:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2507.09820","created_at":"2026-07-05T11:36:40Z"},{"alias_kind":"pith_short_12","alias_value":"OD7T7KHGIJS2","created_at":"2026-07-05T11:36:40Z"},{"alias_kind":"pith_short_16","alias_value":"OD7T7KHGIJS2GD53","created_at":"2026-07-05T11:36:40Z"},{"alias_kind":"pith_short_8","alias_value":"OD7T7KHG","created_at":"2026-07-05T11:36:40Z"}],"graph_snapshots":[{"event_id":"sha256:4427a62b656eb0d0f0c774acbf4c36bb962b79d7307bfeb00ba7711c9471c679","target":"graph","created_at":"2026-07-05T11:36:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2507.09820/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Most safety testing efforts for large language models (LLMs) today focus on evaluating foundation models. However, there is a growing need to evaluate safety at the application level, as components such as system prompts, retrieval pipelines, and guardrails introduce additional factors that significantly influence the overall safety of LLM applications. In this paper, we introduce a practical framework for evaluating application-level safety in LLM systems, validated through real-world deployment across multiple use cases within our organization. The framework consists of two parts: (1) princi","authors_text":"Gabriel Chua, Jessica Foo, Jia Yi Goh, Leanne Tan, Nyx Iskandar, Shaun Khoo","cross_cats":["cs.CY"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-07-13T22:34:20Z","title":"Measuring What Matters: A Framework for Evaluating Safety Risks in Real-World LLM Applications"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2507.09820","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:cbd4ced571ff26c8463d53497d38721a3b2314fee783fdfc92aadd52acdda989","target":"record","created_at":"2026-07-05T11:36:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"5657404bd91ba30897cdded25766eb115edf30f279186fe592c003d016e7b032","cross_cats_sorted":["cs.CY"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-07-13T22:34:20Z","title_canon_sha256":"007259881f5446f46d6028a10e8b3e1eabe662619d7215cb9d9f470d48e4a022"},"schema_version":"1.0","source":{"id":"2507.09820","kind":"arxiv","version":1}},"canonical_sha256":"70ff3fa8e64265a30fbbf19289b53f83232e929707b876510d4cc8fe6b8097df","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"70ff3fa8e64265a30fbbf19289b53f83232e929707b876510d4cc8fe6b8097df","first_computed_at":"2026-07-05T11:36:40.301402Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:36:40.301402Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"UEzdErXvIcf10y/ESpyO7uI3gtZpNogXOT8KYQcpXYnYjh7qJYpmZ2VDv79mO5WG1O1Unorp7w7wHqj2QIdoAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T11:36:40.301941Z","signed_message":"canonical_sha256_bytes"},"source_id":"2507.09820","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:cbd4ced571ff26c8463d53497d38721a3b2314fee783fdfc92aadd52acdda989","sha256:4427a62b656eb0d0f0c774acbf4c36bb962b79d7307bfeb00ba7711c9471c679"],"state_sha256":"ec5859351b0d5a27818ff299c5b18c963f9f62f8544dd5584511d5e22b92da4b"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CEEG6oOGSoSnJ+ge8R4mEp661XEavJbYUSE23JgtQTOoz6zCrbLgjuD76ehxYvvhrlxG5HRq7ky5REwsfa2QAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T12:39:01.202078Z","bundle_sha256":"5eb4cb983c6527bca3fd7f34318ba3fb002b21057092037366b11669051997b7"}}