{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:47UA5CMUFCCSEWP5RHKNU66YRV","short_pith_number":"pith:47UA5CMU","canonical_record":{"source":{"id":"2409.03797","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-09-04T17:53:24Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"7a538ef5b2acf5f160a785f8afe9c1069f2b4a773498384ddbd427412c934a3e","abstract_canon_sha256":"9c461df45a1838676ef0c53016271923c66ec1c07b18bf6d56fbad97a3f4a434"},"schema_version":"1.0"},"canonical_sha256":"e7e80e899428852259fd89d4da7bd88d78375dce5a46a6b0785a9fb7b89b81aa","source":{"kind":"arxiv","id":"2409.03797","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.03797","created_at":"2026-07-05T11:06:23Z"},{"alias_kind":"arxiv_version","alias_value":"2409.03797v3","created_at":"2026-07-05T11:06:23Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.03797","created_at":"2026-07-05T11:06:23Z"},{"alias_kind":"pith_short_12","alias_value":"47UA5CMUFCCS","created_at":"2026-07-05T11:06:23Z"},{"alias_kind":"pith_short_16","alias_value":"47UA5CMUFCCSEWP5","created_at":"2026-07-05T11:06:23Z"},{"alias_kind":"pith_short_8","alias_value":"47UA5CMU","created_at":"2026-07-05T11:06:23Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:47UA5CMUFCCSEWP5RHKNU66YRV","target":"record","payload":{"canonical_record":{"source":{"id":"2409.03797","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-09-04T17:53:24Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"7a538ef5b2acf5f160a785f8afe9c1069f2b4a773498384ddbd427412c934a3e","abstract_canon_sha256":"9c461df45a1838676ef0c53016271923c66ec1c07b18bf6d56fbad97a3f4a434"},"schema_version":"1.0"},"canonical_sha256":"e7e80e899428852259fd89d4da7bd88d78375dce5a46a6b0785a9fb7b89b81aa","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:06:23.121034Z","signature_b64":"W/Iv+j2YBZt6UPy2RltUUytUOmjRf5zdmKOQoL1qBBCmOEpIyfph3KlPbVdgHd++OpPwYXLwyu2SlUNsnHxQBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"e7e80e899428852259fd89d4da7bd88d78375dce5a46a6b0785a9fb7b89b81aa","last_reissued_at":"2026-07-05T11:06:23.120524Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:06:23.120524Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2409.03797","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:06:23Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"VicQUZpq5fZImoht7GQGlCSto/LJNZAawL9Y1ZO0j0gQyDcOcA788i/a6gN8y7YLWBfvlJhHee8KYpY0y/CoCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-16T16:33:14.211536Z"},"content_sha256":"ae37f0080d1505c74ac2cc40c87c1e35b7f41e85ab97655d525b8d41e00806f4","schema_version":"1.0","event_id":"sha256:ae37f0080d1505c74ac2cc40c87c1e35b7f41e85ab97655d525b8d41e00806f4"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:47UA5CMUFCCSEWP5RHKNU66YRV","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"NESTFUL: A Benchmark for Evaluating LLMs on Nested Sequences of API Calls","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.AI","authors_text":"Asim Munawar, Ibrahim Abdelaziz, Kelsey Bradford, Kinjal Basu, Kiran Kate, Luis A. Lastras, Maxwell Crouse, Mayank Agarwal, Pavan Kapanipathi, Sadhana Kumaravel, Saurabh Goyal, Xin Wang, Yara Rizk","submitted_at":"2024-09-04T17:53:24Z","abstract_excerpt":"The resurgence of autonomous agents built using large language models (LLMs) to solve complex real-world tasks has brought increased focus on LLMs' fundamental ability of tool or function calling. At the core of these agents, an LLM must plan, execute, and respond using external tools, APIs, and custom functions. Research on tool calling has gathered momentum, but evaluation benchmarks and datasets representing the complexity of the tasks have lagged behind. In this work, we focus on one such complexity, nested sequencing, with the goal of extending existing benchmarks and evaluation. Specific"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.03797","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2409.03797/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:06:23Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"XHvCN6T5bzU36iPHsivwMn6mAoGOu4jhVfi+2dto1cLm3Y/e/CxvIft2v/6Kmzq9RqLPD2tbvfQGtZzI8vhIBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-16T16:33:14.212049Z"},"content_sha256":"c41695bccc0cb64249c2d3eddf03963b053741f515f4b06384ad57d3ed9c104d","schema_version":"1.0","event_id":"sha256:c41695bccc0cb64249c2d3eddf03963b053741f515f4b06384ad57d3ed9c104d"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/47UA5CMUFCCSEWP5RHKNU66YRV/bundle.json","state_url":"https://pith.science/pith/47UA5CMUFCCSEWP5RHKNU66YRV/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/47UA5CMUFCCSEWP5RHKNU66YRV/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-16T16:33:14Z","links":{"resolver":"https://pith.science/pith/47UA5CMUFCCSEWP5RHKNU66YRV","bundle":"https://pith.science/pith/47UA5CMUFCCSEWP5RHKNU66YRV/bundle.json","state":"https://pith.science/pith/47UA5CMUFCCSEWP5RHKNU66YRV/state.json","well_known_bundle":"https://pith.science/.well-known/pith/47UA5CMUFCCSEWP5RHKNU66YRV/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:47UA5CMUFCCSEWP5RHKNU66YRV","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"9c461df45a1838676ef0c53016271923c66ec1c07b18bf6d56fbad97a3f4a434","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-09-04T17:53:24Z","title_canon_sha256":"7a538ef5b2acf5f160a785f8afe9c1069f2b4a773498384ddbd427412c934a3e"},"schema_version":"1.0","source":{"id":"2409.03797","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.03797","created_at":"2026-07-05T11:06:23Z"},{"alias_kind":"arxiv_version","alias_value":"2409.03797v3","created_at":"2026-07-05T11:06:23Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.03797","created_at":"2026-07-05T11:06:23Z"},{"alias_kind":"pith_short_12","alias_value":"47UA5CMUFCCS","created_at":"2026-07-05T11:06:23Z"},{"alias_kind":"pith_short_16","alias_value":"47UA5CMUFCCSEWP5","created_at":"2026-07-05T11:06:23Z"},{"alias_kind":"pith_short_8","alias_value":"47UA5CMU","created_at":"2026-07-05T11:06:23Z"}],"graph_snapshots":[{"event_id":"sha256:c41695bccc0cb64249c2d3eddf03963b053741f515f4b06384ad57d3ed9c104d","target":"graph","created_at":"2026-07-05T11:06:23Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2409.03797/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The resurgence of autonomous agents built using large language models (LLMs) to solve complex real-world tasks has brought increased focus on LLMs' fundamental ability of tool or function calling. At the core of these agents, an LLM must plan, execute, and respond using external tools, APIs, and custom functions. Research on tool calling has gathered momentum, but evaluation benchmarks and datasets representing the complexity of the tasks have lagged behind. In this work, we focus on one such complexity, nested sequencing, with the goal of extending existing benchmarks and evaluation. Specific","authors_text":"Asim Munawar, Ibrahim Abdelaziz, Kelsey Bradford, Kinjal Basu, Kiran Kate, Luis A. Lastras, Maxwell Crouse, Mayank Agarwal, Pavan Kapanipathi, Sadhana Kumaravel, Saurabh Goyal, Xin Wang, Yara Rizk","cross_cats":["cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-09-04T17:53:24Z","title":"NESTFUL: A Benchmark for Evaluating LLMs on Nested Sequences of API Calls"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.03797","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:ae37f0080d1505c74ac2cc40c87c1e35b7f41e85ab97655d525b8d41e00806f4","target":"record","created_at":"2026-07-05T11:06:23Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"9c461df45a1838676ef0c53016271923c66ec1c07b18bf6d56fbad97a3f4a434","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-09-04T17:53:24Z","title_canon_sha256":"7a538ef5b2acf5f160a785f8afe9c1069f2b4a773498384ddbd427412c934a3e"},"schema_version":"1.0","source":{"id":"2409.03797","kind":"arxiv","version":3}},"canonical_sha256":"e7e80e899428852259fd89d4da7bd88d78375dce5a46a6b0785a9fb7b89b81aa","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"e7e80e899428852259fd89d4da7bd88d78375dce5a46a6b0785a9fb7b89b81aa","first_computed_at":"2026-07-05T11:06:23.120524Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:06:23.120524Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"W/Iv+j2YBZt6UPy2RltUUytUOmjRf5zdmKOQoL1qBBCmOEpIyfph3KlPbVdgHd++OpPwYXLwyu2SlUNsnHxQBw==","signature_status":"signed_v1","signed_at":"2026-07-05T11:06:23.121034Z","signed_message":"canonical_sha256_bytes"},"source_id":"2409.03797","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:ae37f0080d1505c74ac2cc40c87c1e35b7f41e85ab97655d525b8d41e00806f4","sha256:c41695bccc0cb64249c2d3eddf03963b053741f515f4b06384ad57d3ed9c104d"],"state_sha256":"6b93d3d76e92435892f9da88a560cb1c808c3b996805e3d66bc1858f7364cdec"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"PJG2lzR3f6VDyFIz/t31MNiRDTxZ6ElzIKfNy2xWxN5nWZ9hxHyI1nd6TKiE4jiV9M1QZGifi1FChYIuCU2nAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-16T16:33:14.215923Z","bundle_sha256":"653ca3cd9756887b1a99451ee7aea31725c07abb35f1a0b0347a7777283496c8"}}