{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:OD7T7KHGIJS2GD536GJITNJ7QM","short_pith_number":"pith:OD7T7KHG","schema_version":"1.0","canonical_sha256":"70ff3fa8e64265a30fbbf19289b53f83232e929707b876510d4cc8fe6b8097df","source":{"kind":"arxiv","id":"2507.09820","version":1},"attestation_state":"computed","paper":{"title":"Measuring What Matters: A Framework for Evaluating Safety Risks in Real-World LLM Applications","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CY"],"primary_cat":"cs.SE","authors_text":"Gabriel Chua, Jessica Foo, Jia Yi Goh, Leanne Tan, Nyx Iskandar, Shaun Khoo","submitted_at":"2025-07-13T22:34:20Z","abstract_excerpt":"Most safety testing efforts for large language models (LLMs) today focus on evaluating foundation models. However, there is a growing need to evaluate safety at the application level, as components such as system prompts, retrieval pipelines, and guardrails introduce additional factors that significantly influence the overall safety of LLM applications. In this paper, we introduce a practical framework for evaluating application-level safety in LLM systems, validated through real-world deployment across multiple use cases within our organization. The framework consists of two parts: (1) princi"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2507.09820","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-07-13T22:34:20Z","cross_cats_sorted":["cs.CY"],"title_canon_sha256":"007259881f5446f46d6028a10e8b3e1eabe662619d7215cb9d9f470d48e4a022","abstract_canon_sha256":"5657404bd91ba30897cdded25766eb115edf30f279186fe592c003d016e7b032"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:36:40.301941Z","signature_b64":"UEzdErXvIcf10y/ESpyO7uI3gtZpNogXOT8KYQcpXYnYjh7qJYpmZ2VDv79mO5WG1O1Unorp7w7wHqj2QIdoAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"70ff3fa8e64265a30fbbf19289b53f83232e929707b876510d4cc8fe6b8097df","last_reissued_at":"2026-07-05T11:36:40.301402Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:36:40.301402Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Measuring What Matters: A Framework for Evaluating Safety Risks in Real-World LLM Applications","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CY"],"primary_cat":"cs.SE","authors_text":"Gabriel Chua, Jessica Foo, Jia Yi Goh, Leanne Tan, Nyx Iskandar, Shaun Khoo","submitted_at":"2025-07-13T22:34:20Z","abstract_excerpt":"Most safety testing efforts for large language models (LLMs) today focus on evaluating foundation models. However, there is a growing need to evaluate safety at the application level, as components such as system prompts, retrieval pipelines, and guardrails introduce additional factors that significantly influence the overall safety of LLM applications. In this paper, we introduce a practical framework for evaluating application-level safety in LLM systems, validated through real-world deployment across multiple use cases within our organization. The framework consists of two parts: (1) princi"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2507.09820","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2507.09820/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2507.09820","created_at":"2026-07-05T11:36:40.301463+00:00"},{"alias_kind":"arxiv_version","alias_value":"2507.09820v1","created_at":"2026-07-05T11:36:40.301463+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2507.09820","created_at":"2026-07-05T11:36:40.301463+00:00"},{"alias_kind":"pith_short_12","alias_value":"OD7T7KHGIJS2","created_at":"2026-07-05T11:36:40.301463+00:00"},{"alias_kind":"pith_short_16","alias_value":"OD7T7KHGIJS2GD53","created_at":"2026-07-05T11:36:40.301463+00:00"},{"alias_kind":"pith_short_8","alias_value":"OD7T7KHG","created_at":"2026-07-05T11:36:40.301463+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.14152","citing_title":"ROK-FORTRESS: Measuring the Effect of Geopolitical Transcreation for National Security and Public Safety","ref_index":8,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/OD7T7KHGIJS2GD536GJITNJ7QM","json":"https://pith.science/pith/OD7T7KHGIJS2GD536GJITNJ7QM.json","graph_json":"https://pith.science/api/pith-number/OD7T7KHGIJS2GD536GJITNJ7QM/graph.json","events_json":"https://pith.science/api/pith-number/OD7T7KHGIJS2GD536GJITNJ7QM/events.json","paper":"https://pith.science/paper/OD7T7KHG"},"agent_actions":{"view_html":"https://pith.science/pith/OD7T7KHGIJS2GD536GJITNJ7QM","download_json":"https://pith.science/pith/OD7T7KHGIJS2GD536GJITNJ7QM.json","view_paper":"https://pith.science/paper/OD7T7KHG","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2507.09820&json=true","fetch_graph":"https://pith.science/api/pith-number/OD7T7KHGIJS2GD536GJITNJ7QM/graph.json","fetch_events":"https://pith.science/api/pith-number/OD7T7KHGIJS2GD536GJITNJ7QM/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/OD7T7KHGIJS2GD536GJITNJ7QM/action/timestamp_anchor","attest_storage":"https://pith.science/pith/OD7T7KHGIJS2GD536GJITNJ7QM/action/storage_attestation","attest_author":"https://pith.science/pith/OD7T7KHGIJS2GD536GJITNJ7QM/action/author_attestation","sign_citation":"https://pith.science/pith/OD7T7KHGIJS2GD536GJITNJ7QM/action/citation_signature","submit_replication":"https://pith.science/pith/OD7T7KHGIJS2GD536GJITNJ7QM/action/replication_record"}},"created_at":"2026-07-05T11:36:40.301463+00:00","updated_at":"2026-07-05T11:36:40.301463+00:00"}