{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:ZKI55P2ONZVIASEQIOJDYTDH7D","short_pith_number":"pith:ZKI55P2O","schema_version":"1.0","canonical_sha256":"ca91debf4e6e6a80489043923c4c67f8cd419ff5f5d2cd5b352ecae843f06318","source":{"kind":"arxiv","id":"2506.15740","version":2},"attestation_state":"computed","paper":{"title":"SHADE-Arena: Evaluating Sabotage and Monitoring in LLM Agents","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CR","cs.LG"],"primary_cat":"cs.AI","authors_text":"Buck Shlegeris, Chen Bo Calvin Zhang, Henry Sleight, Joe Benton, John Hughes, Jonathan Kutasov, Linda Petrini, Paul Colognese, Teun van der Weij, Tyler Tracy, Xiang Deng, Yuqi Sun","submitted_at":"2025-06-17T15:46:15Z","abstract_excerpt":"As Large Language Models (LLMs) are increasingly deployed as autonomous agents in complex and long horizon settings, it is critical to evaluate their ability to sabotage users by pursuing hidden objectives. We study the ability of frontier LLMs to evade monitoring and achieve harmful hidden goals while completing a wide array of realistic tasks. We evaluate a broad range of frontier LLMs using SHADE (Subtle Harmful Agent Detection & Evaluation)-Arena, the first highly diverse agent evaluation dataset for sabotage and monitoring capabilities of LLM agents. SHADE-Arena consists of complex pairs "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2506.15740","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-06-17T15:46:15Z","cross_cats_sorted":["cs.CR","cs.LG"],"title_canon_sha256":"0ab624098d7cf0672a8207b72e3b32181f532e48e6f0bdfdb1b23339507bfc4a","abstract_canon_sha256":"c1d07cc2d9d82a3a6907c8ebab888e1c767c24319ce895aeacf0109c36e35bdf"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:34:09.578434Z","signature_b64":"MiAiBDQ08EyRbF7JCpIluIAC+X7w7n410803pXt7d/xFSbHM7rb4i1W02lrCugauU3js3YfpdO3yiujwx9NjDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ca91debf4e6e6a80489043923c4c67f8cd419ff5f5d2cd5b352ecae843f06318","last_reissued_at":"2026-07-05T11:34:09.577953Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:34:09.577953Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"SHADE-Arena: Evaluating Sabotage and Monitoring in LLM Agents","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CR","cs.LG"],"primary_cat":"cs.AI","authors_text":"Buck Shlegeris, Chen Bo Calvin Zhang, Henry Sleight, Joe Benton, John Hughes, Jonathan Kutasov, Linda Petrini, Paul Colognese, Teun van der Weij, Tyler Tracy, Xiang Deng, Yuqi Sun","submitted_at":"2025-06-17T15:46:15Z","abstract_excerpt":"As Large Language Models (LLMs) are increasingly deployed as autonomous agents in complex and long horizon settings, it is critical to evaluate their ability to sabotage users by pursuing hidden objectives. We study the ability of frontier LLMs to evade monitoring and achieve harmful hidden goals while completing a wide array of realistic tasks. We evaluate a broad range of frontier LLMs using SHADE (Subtle Harmful Agent Detection & Evaluation)-Arena, the first highly diverse agent evaluation dataset for sabotage and monitoring capabilities of LLM agents. SHADE-Arena consists of complex pairs "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.15740","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.15740/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2506.15740","created_at":"2026-07-05T11:34:09.578009+00:00"},{"alias_kind":"arxiv_version","alias_value":"2506.15740v2","created_at":"2026-07-05T11:34:09.578009+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.15740","created_at":"2026-07-05T11:34:09.578009+00:00"},{"alias_kind":"pith_short_12","alias_value":"ZKI55P2ONZVI","created_at":"2026-07-05T11:34:09.578009+00:00"},{"alias_kind":"pith_short_16","alias_value":"ZKI55P2ONZVIASEQ","created_at":"2026-07-05T11:34:09.578009+00:00"},{"alias_kind":"pith_short_8","alias_value":"ZKI55P2O","created_at":"2026-07-05T11:34:09.578009+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":14,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.25899","citing_title":"Manipulation Is Task-Dependent: A Multi-Axis, Multi-Environment Evaluation of Frontier LLMs","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2606.07157","citing_title":"Think Fast: Estimating No-CoT Task-Completion Time Horizons of Frontier AI Models","ref_index":53,"is_internal_anchor":false},{"citing_arxiv_id":"2606.06529","citing_title":"Attack Selection in Agentic AI Control Evaluations Meaningfully Decreases Safety","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2605.29178","citing_title":"The Best-Laid SCHEMEs: Coordinated Sabotage and Monitoring in Multi-Agent Systems","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2606.07157","citing_title":"Think Fast: Estimating No-CoT Task-Completion Time Horizons of Frontier AI Models","ref_index":53,"is_internal_anchor":false},{"citing_arxiv_id":"2605.16282","citing_title":"Taxonomy and Consistency Analysis of Safety Benchmarks for AI Agents","ref_index":25,"is_internal_anchor":false},{"citing_arxiv_id":"2605.15377","citing_title":"Ensemble Monitoring for AI Control: Diverse Signals Outweigh More Compute","ref_index":51,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08715","citing_title":"AgentForesight: Online Auditing for Early Failure Prediction in Multi-Agent Systems","ref_index":24,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12746","citing_title":"CoT-Guard: Small Models for Strong Monitoring","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08715","citing_title":"AgentForesight: Online Auditing for Early Failure Prediction in Multi-Agent Systems","ref_index":24,"is_internal_anchor":false},{"citing_arxiv_id":"2604.24966","citing_title":"Risk Reporting for Developers' Internal AI Model Use","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2604.23065","citing_title":"What Should Frontier AI Developers Disclose About Internal Deployments?","ref_index":34,"is_internal_anchor":false},{"citing_arxiv_id":"2605.04808","citing_title":"DecodingTrust-Agent Platform (DTap): A Controllable and Interactive Red-Teaming Platform for AI Agents","ref_index":31,"is_internal_anchor":false},{"citing_arxiv_id":"2604.15384","citing_title":"LinuxArena: A Control Setting for AI Agents in Live Production Software Environments","ref_index":8,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/ZKI55P2ONZVIASEQIOJDYTDH7D","json":"https://pith.science/pith/ZKI55P2ONZVIASEQIOJDYTDH7D.json","graph_json":"https://pith.science/api/pith-number/ZKI55P2ONZVIASEQIOJDYTDH7D/graph.json","events_json":"https://pith.science/api/pith-number/ZKI55P2ONZVIASEQIOJDYTDH7D/events.json","paper":"https://pith.science/paper/ZKI55P2O"},"agent_actions":{"view_html":"https://pith.science/pith/ZKI55P2ONZVIASEQIOJDYTDH7D","download_json":"https://pith.science/pith/ZKI55P2ONZVIASEQIOJDYTDH7D.json","view_paper":"https://pith.science/paper/ZKI55P2O","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2506.15740&json=true","fetch_graph":"https://pith.science/api/pith-number/ZKI55P2ONZVIASEQIOJDYTDH7D/graph.json","fetch_events":"https://pith.science/api/pith-number/ZKI55P2ONZVIASEQIOJDYTDH7D/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/ZKI55P2ONZVIASEQIOJDYTDH7D/action/timestamp_anchor","attest_storage":"https://pith.science/pith/ZKI55P2ONZVIASEQIOJDYTDH7D/action/storage_attestation","attest_author":"https://pith.science/pith/ZKI55P2ONZVIASEQIOJDYTDH7D/action/author_attestation","sign_citation":"https://pith.science/pith/ZKI55P2ONZVIASEQIOJDYTDH7D/action/citation_signature","submit_replication":"https://pith.science/pith/ZKI55P2ONZVIASEQIOJDYTDH7D/action/replication_record"}},"created_at":"2026-07-05T11:34:09.578009+00:00","updated_at":"2026-07-05T11:34:09.578009+00:00"}