{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:ANO54TSVLIWZVVDF7H55CNOQ2L","short_pith_number":"pith:ANO54TSV","canonical_record":{"source":{"id":"2503.08669","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-11T17:53:02Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"c036efa630d311699a0fe4d39bf623b6383ac84f5398d28928b7c0c1c5e01172","abstract_canon_sha256":"6709e00b06a5b5c5c20d85024eedcc73b4a97d44f72b3fe6d8b0b5755aa8dd29"},"schema_version":"1.0"},"canonical_sha256":"035dde4e555a2d9ad465f9fbd135d0d2c0b917ab9683c8f5c609152072653c4e","source":{"kind":"arxiv","id":"2503.08669","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.08669","created_at":"2026-07-05T11:23:09Z"},{"alias_kind":"arxiv_version","alias_value":"2503.08669v2","created_at":"2026-07-05T11:23:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.08669","created_at":"2026-07-05T11:23:09Z"},{"alias_kind":"pith_short_12","alias_value":"ANO54TSVLIWZ","created_at":"2026-07-05T11:23:09Z"},{"alias_kind":"pith_short_16","alias_value":"ANO54TSVLIWZVVDF","created_at":"2026-07-05T11:23:09Z"},{"alias_kind":"pith_short_8","alias_value":"ANO54TSV","created_at":"2026-07-05T11:23:09Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:ANO54TSVLIWZVVDF7H55CNOQ2L","target":"record","payload":{"canonical_record":{"source":{"id":"2503.08669","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-11T17:53:02Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"c036efa630d311699a0fe4d39bf623b6383ac84f5398d28928b7c0c1c5e01172","abstract_canon_sha256":"6709e00b06a5b5c5c20d85024eedcc73b4a97d44f72b3fe6d8b0b5755aa8dd29"},"schema_version":"1.0"},"canonical_sha256":"035dde4e555a2d9ad465f9fbd135d0d2c0b917ab9683c8f5c609152072653c4e","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:23:09.424745Z","signature_b64":"kHDy/xdcCO0WZEVqADQU52dNIgqu4C/XqI+5B7elJLwXGGdolnK5JHX/4CwuMPIcsuqvbWsoHxajniAUiWC/DA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"035dde4e555a2d9ad465f9fbd135d0d2c0b917ab9683c8f5c609152072653c4e","last_reissued_at":"2026-07-05T11:23:09.424206Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:23:09.424206Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2503.08669","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:23:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"I40mdiqmQFpUp+Fzv26vuEp+1XQZJUK4SLlcNgdvOdkenHKoN/B7A/pmrRez1sgj3xVS/4fpYzzUAXom9+SeCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T01:33:09.947280Z"},"content_sha256":"a0d1715c9589050d884b686e33f0ad2365cc05488667edd3ffe448701e968fe1","schema_version":"1.0","event_id":"sha256:a0d1715c9589050d884b686e33f0ad2365cc05488667edd3ffe448701e968fe1"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:ANO54TSVLIWZVVDF7H55CNOQ2L","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"SOPBench: Evaluating Language Agents at Following Standard Operating Procedures and Constraints","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Antonis Antoniades, Chi Wang, Jiangtian Wang, Kaijie Zhu, Nathan Zhang, Shinda Huang, Sirui Zeng, Wenyue Hua, William Yang Wang, Xifeng Yan, Zekun Li","submitted_at":"2025-03-11T17:53:02Z","abstract_excerpt":"As language agents increasingly automate critical tasks, their ability to follow domain-specific standard operating procedures (SOPs), policies, and constraints when taking actions and making tool calls becomes essential yet remains underexplored. To address this gap, we develop an automated evaluation pipeline SOPBench with: (1) executable environments containing 167 tools/functions across seven customer service domains with service-specific SOPs and rule-based verifiers, (2) an automated test generation framework producing over 900 verified test cases, and (3) an automated evaluation framewo"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.08669","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2503.08669/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:23:09Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"B4N0RRHMJ9upjHw5zMKluuyxsVjLLE2iQkiN6KWaezfkYkG3SD6jOymzvW5o7MFVkrPD0R1Z9HRhf2MwLgWuDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T01:33:09.948031Z"},"content_sha256":"94ce058ec79b1b89210c9ff6a60993d0684943d600b461199489c26ef556374d","schema_version":"1.0","event_id":"sha256:94ce058ec79b1b89210c9ff6a60993d0684943d600b461199489c26ef556374d"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ANO54TSVLIWZVVDF7H55CNOQ2L/bundle.json","state_url":"https://pith.science/pith/ANO54TSVLIWZVVDF7H55CNOQ2L/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ANO54TSVLIWZVVDF7H55CNOQ2L/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-01T01:33:09Z","links":{"resolver":"https://pith.science/pith/ANO54TSVLIWZVVDF7H55CNOQ2L","bundle":"https://pith.science/pith/ANO54TSVLIWZVVDF7H55CNOQ2L/bundle.json","state":"https://pith.science/pith/ANO54TSVLIWZVVDF7H55CNOQ2L/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ANO54TSVLIWZVVDF7H55CNOQ2L/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:ANO54TSVLIWZVVDF7H55CNOQ2L","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"6709e00b06a5b5c5c20d85024eedcc73b4a97d44f72b3fe6d8b0b5755aa8dd29","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-11T17:53:02Z","title_canon_sha256":"c036efa630d311699a0fe4d39bf623b6383ac84f5398d28928b7c0c1c5e01172"},"schema_version":"1.0","source":{"id":"2503.08669","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.08669","created_at":"2026-07-05T11:23:09Z"},{"alias_kind":"arxiv_version","alias_value":"2503.08669v2","created_at":"2026-07-05T11:23:09Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.08669","created_at":"2026-07-05T11:23:09Z"},{"alias_kind":"pith_short_12","alias_value":"ANO54TSVLIWZ","created_at":"2026-07-05T11:23:09Z"},{"alias_kind":"pith_short_16","alias_value":"ANO54TSVLIWZVVDF","created_at":"2026-07-05T11:23:09Z"},{"alias_kind":"pith_short_8","alias_value":"ANO54TSV","created_at":"2026-07-05T11:23:09Z"}],"graph_snapshots":[{"event_id":"sha256:94ce058ec79b1b89210c9ff6a60993d0684943d600b461199489c26ef556374d","target":"graph","created_at":"2026-07-05T11:23:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2503.08669/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"As language agents increasingly automate critical tasks, their ability to follow domain-specific standard operating procedures (SOPs), policies, and constraints when taking actions and making tool calls becomes essential yet remains underexplored. To address this gap, we develop an automated evaluation pipeline SOPBench with: (1) executable environments containing 167 tools/functions across seven customer service domains with service-specific SOPs and rule-based verifiers, (2) an automated test generation framework producing over 900 verified test cases, and (3) an automated evaluation framewo","authors_text":"Antonis Antoniades, Chi Wang, Jiangtian Wang, Kaijie Zhu, Nathan Zhang, Shinda Huang, Sirui Zeng, Wenyue Hua, William Yang Wang, Xifeng Yan, Zekun Li","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-11T17:53:02Z","title":"SOPBench: Evaluating Language Agents at Following Standard Operating Procedures and Constraints"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.08669","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a0d1715c9589050d884b686e33f0ad2365cc05488667edd3ffe448701e968fe1","target":"record","created_at":"2026-07-05T11:23:09Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"6709e00b06a5b5c5c20d85024eedcc73b4a97d44f72b3fe6d8b0b5755aa8dd29","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-11T17:53:02Z","title_canon_sha256":"c036efa630d311699a0fe4d39bf623b6383ac84f5398d28928b7c0c1c5e01172"},"schema_version":"1.0","source":{"id":"2503.08669","kind":"arxiv","version":2}},"canonical_sha256":"035dde4e555a2d9ad465f9fbd135d0d2c0b917ab9683c8f5c609152072653c4e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"035dde4e555a2d9ad465f9fbd135d0d2c0b917ab9683c8f5c609152072653c4e","first_computed_at":"2026-07-05T11:23:09.424206Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:23:09.424206Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"kHDy/xdcCO0WZEVqADQU52dNIgqu4C/XqI+5B7elJLwXGGdolnK5JHX/4CwuMPIcsuqvbWsoHxajniAUiWC/DA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:23:09.424745Z","signed_message":"canonical_sha256_bytes"},"source_id":"2503.08669","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a0d1715c9589050d884b686e33f0ad2365cc05488667edd3ffe448701e968fe1","sha256:94ce058ec79b1b89210c9ff6a60993d0684943d600b461199489c26ef556374d"],"state_sha256":"1b806b1127a2ff217e302e8d45f07199b3122216025870edb2c005a80e02ea15"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QV3c1WKmZOcjI8T5DcUZ+sx8DIlSroP2jF/Keb7gVe/18VlciZOUATJTURs6le2jGMCS9GifH6oBc3YTb6ylBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-01T01:33:09.954050Z","bundle_sha256":"1d6cda963ad125bba755882656c18da42ccde4980e10edf9f839aa14f6dc7d1d"}}