{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:MEBLKTBRUCUZJXC2ASNWFPG3LI","short_pith_number":"pith:MEBLKTBR","canonical_record":{"source":{"id":"2411.13547","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-11-20T18:56:22Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"ac8157c72627284b9d000983beefa5e51d5024c0e101405897782073c48ffc9c","abstract_canon_sha256":"d7c1bf8ea7ff4cb96d9ec67aa1d20dbf29a3b3b3ee942c36556fd53b2b57d952"},"schema_version":"1.0"},"canonical_sha256":"6102b54c31a0a994dc5a049b62bcdb5a1376432f485a41b7067a71e797c55a4a","source":{"kind":"arxiv","id":"2411.13547","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.13547","created_at":"2026-07-05T11:27:21Z"},{"alias_kind":"arxiv_version","alias_value":"2411.13547v2","created_at":"2026-07-05T11:27:21Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.13547","created_at":"2026-07-05T11:27:21Z"},{"alias_kind":"pith_short_12","alias_value":"MEBLKTBRUCUZ","created_at":"2026-07-05T11:27:21Z"},{"alias_kind":"pith_short_16","alias_value":"MEBLKTBRUCUZJXC2","created_at":"2026-07-05T11:27:21Z"},{"alias_kind":"pith_short_8","alias_value":"MEBLKTBR","created_at":"2026-07-05T11:27:21Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:MEBLKTBRUCUZJXC2ASNWFPG3LI","target":"record","payload":{"canonical_record":{"source":{"id":"2411.13547","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-11-20T18:56:22Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"ac8157c72627284b9d000983beefa5e51d5024c0e101405897782073c48ffc9c","abstract_canon_sha256":"d7c1bf8ea7ff4cb96d9ec67aa1d20dbf29a3b3b3ee942c36556fd53b2b57d952"},"schema_version":"1.0"},"canonical_sha256":"6102b54c31a0a994dc5a049b62bcdb5a1376432f485a41b7067a71e797c55a4a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:27:21.876179Z","signature_b64":"2UZhLudQbXY7X172SYHUpcKIa/iZapFxFKhp8nAnJYylUlPl7aXSZNoJGUOkOHGZovP/V/toUk+mLieXauKpBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6102b54c31a0a994dc5a049b62bcdb5a1376432f485a41b7067a71e797c55a4a","last_reissued_at":"2026-07-05T11:27:21.875654Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:27:21.875654Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2411.13547","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:27:21Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"74ATHVuN1FYzwK7s6p0cw6rmA3BSM4iI6n2dWs+N0pFd4mpMGhKuNB08c8xeduIsuvHNaE2qEmcmR5ej1zQyCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T18:41:40.050028Z"},"content_sha256":"1c3d45b96562af49a966eafdd57787186cfca3e91f7c705324642c635f8335d2","schema_version":"1.0","event_id":"sha256:1c3d45b96562af49a966eafdd57787186cfca3e91f7c705324642c635f8335d2"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:MEBLKTBRUCUZJXC2ASNWFPG3LI","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"ToolScan: A Benchmark for Characterizing Errors in Tool-Use LLMs","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.SE","authors_text":"Akshara Prabhakar, Caiming Xiong, Huan Wang, Jianguo Zhang, Juan Carlos Niebles, Juntao Tan, Liangwei Yang, Ming Zhu, Rithesh Murthy, Shelby Heinecke, Shirley Kokane, Silivo Savarese, Thai Hoang, Tian Lan, Tulika Awalgaonkar, Weiran Yao, Zhiwei Liu, Zuxin Liu","submitted_at":"2024-11-20T18:56:22Z","abstract_excerpt":"Evaluating Large Language Models (LLMs) is one of the most critical aspects of building a performant compound AI system. Since the output from LLMs propagate to downstream steps, identifying LLM errors is crucial to system performance. A common task for LLMs in AI systems is tool use. While there are several benchmark environments for evaluating LLMs on this task, they typically only give a success rate without any explanation of the failure cases. To solve this problem, we introduce TOOLSCAN, a new benchmark to identify error patterns in LLM output on tool-use tasks. Our benchmark data set co"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.13547","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2411.13547/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:27:21Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"PcgHT0iNxidMfdRkLPViHCiDDEvACghT6wqnCyT7Sg5/EPF/eE4EwST5pxewsaPCMEsDcIAv+aynqTf0EGKsBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T18:41:40.050424Z"},"content_sha256":"18ececd396049da952dc950076d61464b279720201699a3bc9851730e52aa880","schema_version":"1.0","event_id":"sha256:18ececd396049da952dc950076d61464b279720201699a3bc9851730e52aa880"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/MEBLKTBRUCUZJXC2ASNWFPG3LI/bundle.json","state_url":"https://pith.science/pith/MEBLKTBRUCUZJXC2ASNWFPG3LI/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/MEBLKTBRUCUZJXC2ASNWFPG3LI/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T18:41:40Z","links":{"resolver":"https://pith.science/pith/MEBLKTBRUCUZJXC2ASNWFPG3LI","bundle":"https://pith.science/pith/MEBLKTBRUCUZJXC2ASNWFPG3LI/bundle.json","state":"https://pith.science/pith/MEBLKTBRUCUZJXC2ASNWFPG3LI/state.json","well_known_bundle":"https://pith.science/.well-known/pith/MEBLKTBRUCUZJXC2ASNWFPG3LI/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:MEBLKTBRUCUZJXC2ASNWFPG3LI","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"d7c1bf8ea7ff4cb96d9ec67aa1d20dbf29a3b3b3ee942c36556fd53b2b57d952","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-11-20T18:56:22Z","title_canon_sha256":"ac8157c72627284b9d000983beefa5e51d5024c0e101405897782073c48ffc9c"},"schema_version":"1.0","source":{"id":"2411.13547","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.13547","created_at":"2026-07-05T11:27:21Z"},{"alias_kind":"arxiv_version","alias_value":"2411.13547v2","created_at":"2026-07-05T11:27:21Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.13547","created_at":"2026-07-05T11:27:21Z"},{"alias_kind":"pith_short_12","alias_value":"MEBLKTBRUCUZ","created_at":"2026-07-05T11:27:21Z"},{"alias_kind":"pith_short_16","alias_value":"MEBLKTBRUCUZJXC2","created_at":"2026-07-05T11:27:21Z"},{"alias_kind":"pith_short_8","alias_value":"MEBLKTBR","created_at":"2026-07-05T11:27:21Z"}],"graph_snapshots":[{"event_id":"sha256:18ececd396049da952dc950076d61464b279720201699a3bc9851730e52aa880","target":"graph","created_at":"2026-07-05T11:27:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2411.13547/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Evaluating Large Language Models (LLMs) is one of the most critical aspects of building a performant compound AI system. Since the output from LLMs propagate to downstream steps, identifying LLM errors is crucial to system performance. A common task for LLMs in AI systems is tool use. While there are several benchmark environments for evaluating LLMs on this task, they typically only give a success rate without any explanation of the failure cases. To solve this problem, we introduce TOOLSCAN, a new benchmark to identify error patterns in LLM output on tool-use tasks. Our benchmark data set co","authors_text":"Akshara Prabhakar, Caiming Xiong, Huan Wang, Jianguo Zhang, Juan Carlos Niebles, Juntao Tan, Liangwei Yang, Ming Zhu, Rithesh Murthy, Shelby Heinecke, Shirley Kokane, Silivo Savarese, Thai Hoang, Tian Lan, Tulika Awalgaonkar, Weiran Yao, Zhiwei Liu, Zuxin Liu","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-11-20T18:56:22Z","title":"ToolScan: A Benchmark for Characterizing Errors in Tool-Use LLMs"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.13547","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1c3d45b96562af49a966eafdd57787186cfca3e91f7c705324642c635f8335d2","target":"record","created_at":"2026-07-05T11:27:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"d7c1bf8ea7ff4cb96d9ec67aa1d20dbf29a3b3b3ee942c36556fd53b2b57d952","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-11-20T18:56:22Z","title_canon_sha256":"ac8157c72627284b9d000983beefa5e51d5024c0e101405897782073c48ffc9c"},"schema_version":"1.0","source":{"id":"2411.13547","kind":"arxiv","version":2}},"canonical_sha256":"6102b54c31a0a994dc5a049b62bcdb5a1376432f485a41b7067a71e797c55a4a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6102b54c31a0a994dc5a049b62bcdb5a1376432f485a41b7067a71e797c55a4a","first_computed_at":"2026-07-05T11:27:21.875654Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:27:21.875654Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"2UZhLudQbXY7X172SYHUpcKIa/iZapFxFKhp8nAnJYylUlPl7aXSZNoJGUOkOHGZovP/V/toUk+mLieXauKpBg==","signature_status":"signed_v1","signed_at":"2026-07-05T11:27:21.876179Z","signed_message":"canonical_sha256_bytes"},"source_id":"2411.13547","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1c3d45b96562af49a966eafdd57787186cfca3e91f7c705324642c635f8335d2","sha256:18ececd396049da952dc950076d61464b279720201699a3bc9851730e52aa880"],"state_sha256":"d9ba76fc21a61af7b92fa3f2fb953b026c1aa8b4deece02300e97e9309a3476a"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"n/pu3EjDI0tzbNd08sZyLn4SZCN538LJu72BEkONBrclsLWVY+DYHC7fA/S9cv7njqjnBsHeWJnGngbED309AQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T18:41:40.052449Z","bundle_sha256":"12742982a3972b482e5fcd342e662ab341f5026487e692df9214b82631428ff9"}}