{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2026:WD3B2JZE5BRKBJ4OYJCEYBTOLR","short_pith_number":"pith:WD3B2JZE","canonical_record":{"source":{"id":"2604.06367","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CR","submitted_at":"2026-04-07T18:43:21Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"949d939d907c70ceab52a9974ce5db794a207f44492c520620254615be517b1f","abstract_canon_sha256":"7b6fae091517999ac591814da14f8229b62cd466cedbbf474ebf300111774951"},"schema_version":"1.0"},"canonical_sha256":"b0f61d2724e862a0a78ec2444c066e5c46791720d4b1778e8f07514391ed36d5","source":{"kind":"arxiv","id":"2604.06367","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2604.06367","created_at":"2026-06-19T16:11:22Z"},{"alias_kind":"arxiv_version","alias_value":"2604.06367v2","created_at":"2026-06-19T16:11:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2604.06367","created_at":"2026-06-19T16:11:22Z"},{"alias_kind":"pith_short_12","alias_value":"WD3B2JZE5BRK","created_at":"2026-06-19T16:11:22Z"},{"alias_kind":"pith_short_16","alias_value":"WD3B2JZE5BRKBJ4O","created_at":"2026-06-19T16:11:22Z"},{"alias_kind":"pith_short_8","alias_value":"WD3B2JZE","created_at":"2026-06-19T16:11:22Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2026:WD3B2JZE5BRKBJ4OYJCEYBTOLR","target":"record","payload":{"canonical_record":{"source":{"id":"2604.06367","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CR","submitted_at":"2026-04-07T18:43:21Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"949d939d907c70ceab52a9974ce5db794a207f44492c520620254615be517b1f","abstract_canon_sha256":"7b6fae091517999ac591814da14f8229b62cd466cedbbf474ebf300111774951"},"schema_version":"1.0"},"canonical_sha256":"b0f61d2724e862a0a78ec2444c066e5c46791720d4b1778e8f07514391ed36d5","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-06-19T16:11:22.927981Z","signature_b64":"GXL+dDIq3GYHlDFWNogVqxi5iwMzgXO6yjGuevRDsUhCb4WC6kRg2GuPKYalRi69iL0HeCKK0BAXImXEobzsDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b0f61d2724e862a0a78ec2444c066e5c46791720d4b1778e8f07514391ed36d5","last_reissued_at":"2026-06-19T16:11:22.927556Z","signature_status":"signed_v1","first_computed_at":"2026-06-19T16:11:22.927556Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2604.06367","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-06-19T16:11:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"oAj5ywVXKrHohq9nYvEO8kzwkQ+RUpMis8sjLQAFpQkeVBWbxaat91NsqcukgVDBFUDRVcA/nylDmf/MASPUBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-14T21:13:44.515367Z"},"content_sha256":"281edacff4eb649fec8ffd196a00cdc2ce3f1753fa621cde57835af3ae033cf4","schema_version":"1.0","event_id":"sha256:281edacff4eb649fec8ffd196a00cdc2ce3f1753fa621cde57835af3ae033cf4"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2026:WD3B2JZE5BRKBJ4OYJCEYBTOLR","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"WebSP-Eval: Evaluating Web Agents on Website Security and Privacy Tasks","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"Current web agents fail more than 45 percent of the time on security and privacy tasks that use stateful UI elements such as toggles and checkboxes.","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CR","authors_text":"Asmit Nayak, Basieem Siddique, Guruprasad Viswanathan Ramesh, Kassem Fawaz","submitted_at":"2026-04-07T18:43:21Z","abstract_excerpt":"Web agents automate browser tasks, ranging from simple form completion to complex workflows like ordering groceries. While current benchmarks evaluate general-purpose performance~(e.g., WebArena) or safety against malicious actions~(e.g., SafeArena), no existing framework assesses an agent's ability to successfully execute user-facing website security and privacy tasks, such as managing cookie preferences, configuring privacy-sensitive account settings, or revoking inactive sessions.\n  To address this gap, we introduce WebSP-Eval, an evaluation framework for measuring web agent performance on "},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"Our evaluation reveals that current models suffer from limited autonomous exploration capabilities to reliably solve website security and privacy tasks, and struggle with specific task categories and websites. Crucially, we identify stateful UI elements such as toggles and checkboxes are a primary reason for agent failure, failing at a rate of more than 45% in tasks containing these elements across many models.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"The 200 manually crafted task instances across 28 websites are representative of the diversity and frequency of real-world user-facing security and privacy tasks, and the custom Chrome extension accurately maintains consistent initial states without introducing evaluation artifacts or biases.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"WebSP-Eval shows that multimodal LLM-based web agents fail more than 45% of the time on security and privacy tasks involving stateful UI elements such as toggles and checkboxes.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"Current web agents fail more than 45 percent of the time on security and privacy tasks that use stateful UI elements such as toggles and checkboxes.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"dc2a8e9ecefb346ad4ac9cbf3e074f338b8f0cd3b83e1baf0bcc695aa6199b03"},"source":{"id":"2604.06367","kind":"arxiv","version":2},"verdict":{"id":"55d288ec-e3b6-4ba0-a4e3-766c273b5255","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-10T18:44:14.179559Z","strongest_claim":"Our evaluation reveals that current models suffer from limited autonomous exploration capabilities to reliably solve website security and privacy tasks, and struggle with specific task categories and websites. Crucially, we identify stateful UI elements such as toggles and checkboxes are a primary reason for agent failure, failing at a rate of more than 45% in tasks containing these elements across many models.","one_line_summary":"WebSP-Eval shows that multimodal LLM-based web agents fail more than 45% of the time on security and privacy tasks involving stateful UI elements such as toggles and checkboxes.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"The 200 manually crafted task instances across 28 websites are representative of the diversity and frequency of real-world user-facing security and privacy tasks, and the custom Chrome extension accurately maintains consistent initial states without introducing evaluation artifacts or biases.","pith_extraction_headline":"Current web agents fail more than 45 percent of the time on security and privacy tasks that use stateful UI elements such as toggles and checkboxes."},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2604.06367/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":"55d288ec-e3b6-4ba0-a4e3-766c273b5255"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-06-19T16:11:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"gIoiLknNOAUDc3z4PMb5rodHaOFT+IIQpJX8ZKXdh5AgIfFGrU6QsoKvIQv/1hBl2DlJH2yl3VyHBb7xJCDqCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-14T21:13:44.515842Z"},"content_sha256":"2a1f642dd9a4b2c3a43bed59cdb738ba6ced398b2d288a2f075e0c76be41a09f","schema_version":"1.0","event_id":"sha256:2a1f642dd9a4b2c3a43bed59cdb738ba6ced398b2d288a2f075e0c76be41a09f"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/WD3B2JZE5BRKBJ4OYJCEYBTOLR/bundle.json","state_url":"https://pith.science/pith/WD3B2JZE5BRKBJ4OYJCEYBTOLR/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/WD3B2JZE5BRKBJ4OYJCEYBTOLR/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-14T21:13:44Z","links":{"resolver":"https://pith.science/pith/WD3B2JZE5BRKBJ4OYJCEYBTOLR","bundle":"https://pith.science/pith/WD3B2JZE5BRKBJ4OYJCEYBTOLR/bundle.json","state":"https://pith.science/pith/WD3B2JZE5BRKBJ4OYJCEYBTOLR/state.json","well_known_bundle":"https://pith.science/.well-known/pith/WD3B2JZE5BRKBJ4OYJCEYBTOLR/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2026:WD3B2JZE5BRKBJ4OYJCEYBTOLR","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7b6fae091517999ac591814da14f8229b62cd466cedbbf474ebf300111774951","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CR","submitted_at":"2026-04-07T18:43:21Z","title_canon_sha256":"949d939d907c70ceab52a9974ce5db794a207f44492c520620254615be517b1f"},"schema_version":"1.0","source":{"id":"2604.06367","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2604.06367","created_at":"2026-06-19T16:11:22Z"},{"alias_kind":"arxiv_version","alias_value":"2604.06367v2","created_at":"2026-06-19T16:11:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2604.06367","created_at":"2026-06-19T16:11:22Z"},{"alias_kind":"pith_short_12","alias_value":"WD3B2JZE5BRK","created_at":"2026-06-19T16:11:22Z"},{"alias_kind":"pith_short_16","alias_value":"WD3B2JZE5BRKBJ4O","created_at":"2026-06-19T16:11:22Z"},{"alias_kind":"pith_short_8","alias_value":"WD3B2JZE","created_at":"2026-06-19T16:11:22Z"}],"graph_snapshots":[{"event_id":"sha256:2a1f642dd9a4b2c3a43bed59cdb738ba6ced398b2d288a2f075e0c76be41a09f","target":"graph","created_at":"2026-06-19T16:11:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"Our evaluation reveals that current models suffer from limited autonomous exploration capabilities to reliably solve website security and privacy tasks, and struggle with specific task categories and websites. Crucially, we identify stateful UI elements such as toggles and checkboxes are a primary reason for agent failure, failing at a rate of more than 45% in tasks containing these elements across many models."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"The 200 manually crafted task instances across 28 websites are representative of the diversity and frequency of real-world user-facing security and privacy tasks, and the custom Chrome extension accurately maintains consistent initial states without introducing evaluation artifacts or biases."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"WebSP-Eval shows that multimodal LLM-based web agents fail more than 45% of the time on security and privacy tasks involving stateful UI elements such as toggles and checkboxes."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"Current web agents fail more than 45 percent of the time on security and privacy tasks that use stateful UI elements such as toggles and checkboxes."}],"snapshot_sha256":"dc2a8e9ecefb346ad4ac9cbf3e074f338b8f0cd3b83e1baf0bcc695aa6199b03"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2604.06367/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Web agents automate browser tasks, ranging from simple form completion to complex workflows like ordering groceries. While current benchmarks evaluate general-purpose performance~(e.g., WebArena) or safety against malicious actions~(e.g., SafeArena), no existing framework assesses an agent's ability to successfully execute user-facing website security and privacy tasks, such as managing cookie preferences, configuring privacy-sensitive account settings, or revoking inactive sessions.\n  To address this gap, we introduce WebSP-Eval, an evaluation framework for measuring web agent performance on ","authors_text":"Asmit Nayak, Basieem Siddique, Guruprasad Viswanathan Ramesh, Kassem Fawaz","cross_cats":["cs.AI","cs.LG"],"headline":"Current web agents fail more than 45 percent of the time on security and privacy tasks that use stateful UI elements such as toggles and checkboxes.","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CR","submitted_at":"2026-04-07T18:43:21Z","title":"WebSP-Eval: Evaluating Web Agents on Website Security and Privacy Tasks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2604.06367","kind":"arxiv","version":2},"verdict":{"created_at":"2026-05-10T18:44:14.179559Z","id":"55d288ec-e3b6-4ba0-a4e3-766c273b5255","model_set":{"reader":"grok-4.3"},"one_line_summary":"WebSP-Eval shows that multimodal LLM-based web agents fail more than 45% of the time on security and privacy tasks involving stateful UI elements such as toggles and checkboxes.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"Current web agents fail more than 45 percent of the time on security and privacy tasks that use stateful UI elements such as toggles and checkboxes.","strongest_claim":"Our evaluation reveals that current models suffer from limited autonomous exploration capabilities to reliably solve website security and privacy tasks, and struggle with specific task categories and websites. Crucially, we identify stateful UI elements such as toggles and checkboxes are a primary reason for agent failure, failing at a rate of more than 45% in tasks containing these elements across many models.","weakest_assumption":"The 200 manually crafted task instances across 28 websites are representative of the diversity and frequency of real-world user-facing security and privacy tasks, and the custom Chrome extension accurately maintains consistent initial states without introducing evaluation artifacts or biases."}},"verdict_id":"55d288ec-e3b6-4ba0-a4e3-766c273b5255"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:281edacff4eb649fec8ffd196a00cdc2ce3f1753fa621cde57835af3ae033cf4","target":"record","created_at":"2026-06-19T16:11:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7b6fae091517999ac591814da14f8229b62cd466cedbbf474ebf300111774951","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CR","submitted_at":"2026-04-07T18:43:21Z","title_canon_sha256":"949d939d907c70ceab52a9974ce5db794a207f44492c520620254615be517b1f"},"schema_version":"1.0","source":{"id":"2604.06367","kind":"arxiv","version":2}},"canonical_sha256":"b0f61d2724e862a0a78ec2444c066e5c46791720d4b1778e8f07514391ed36d5","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"b0f61d2724e862a0a78ec2444c066e5c46791720d4b1778e8f07514391ed36d5","first_computed_at":"2026-06-19T16:11:22.927556Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-06-19T16:11:22.927556Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"GXL+dDIq3GYHlDFWNogVqxi5iwMzgXO6yjGuevRDsUhCb4WC6kRg2GuPKYalRi69iL0HeCKK0BAXImXEobzsDQ==","signature_status":"signed_v1","signed_at":"2026-06-19T16:11:22.927981Z","signed_message":"canonical_sha256_bytes"},"source_id":"2604.06367","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:281edacff4eb649fec8ffd196a00cdc2ce3f1753fa621cde57835af3ae033cf4","sha256:2a1f642dd9a4b2c3a43bed59cdb738ba6ced398b2d288a2f075e0c76be41a09f"],"state_sha256":"295f840577d84ed5650460b449b57de5a2a2257e928ad39b0a55d585218e8c6f"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"lnFlzhcbxcexpbOZiz9Xks913o6x9/y9WtDzW3tdeG86yzxHnc8UhPB43MvNxa0KMcLsCBBCbroN7X9/r5mUDA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-14T21:13:44.518670Z","bundle_sha256":"6af7bd231bf5f0a2c8e794b886b7948f68daac1e9e9c2824d855b0e935ed93a6"}}