{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:BQND6NDDR4MBD6M6VZRKULHREI","short_pith_number":"pith:BQND6NDD","canonical_record":{"source":{"id":"2409.15934","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-24T09:57:43Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"775c3e9a3e969d79aa00a7733ff74288e677cf0a56d4b5eee1ea6ac82804acce","abstract_canon_sha256":"107db695b677e2f35b3b2b34b3776ed4a4e87d33e35d2107ab51e1cc998ba46c"},"schema_version":"1.0"},"canonical_sha256":"0c1a3f34638f1811f99eae62aa2cf122199e6313d0f0cd489d97006091b399d6","source":{"kind":"arxiv","id":"2409.15934","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.15934","created_at":"2026-07-05T09:18:30Z"},{"alias_kind":"arxiv_version","alias_value":"2409.15934v2","created_at":"2026-07-05T09:18:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.15934","created_at":"2026-07-05T09:18:30Z"},{"alias_kind":"pith_short_12","alias_value":"BQND6NDDR4MB","created_at":"2026-07-05T09:18:30Z"},{"alias_kind":"pith_short_16","alias_value":"BQND6NDDR4MBD6M6","created_at":"2026-07-05T09:18:30Z"},{"alias_kind":"pith_short_8","alias_value":"BQND6NDD","created_at":"2026-07-05T09:18:30Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:BQND6NDDR4MBD6M6VZRKULHREI","target":"record","payload":{"canonical_record":{"source":{"id":"2409.15934","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-24T09:57:43Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"775c3e9a3e969d79aa00a7733ff74288e677cf0a56d4b5eee1ea6ac82804acce","abstract_canon_sha256":"107db695b677e2f35b3b2b34b3776ed4a4e87d33e35d2107ab51e1cc998ba46c"},"schema_version":"1.0"},"canonical_sha256":"0c1a3f34638f1811f99eae62aa2cf122199e6313d0f0cd489d97006091b399d6","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:18:30.215876Z","signature_b64":"BzjgJJ9SJahzBhpJuyUiSXlZcO+8BjJdls8hNL/dRFLC9Ja1I+eSrNIwnA3NTDZnwM7kevc1XQIonowYYrALDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0c1a3f34638f1811f99eae62aa2cf122199e6313d0f0cd489d97006091b399d6","last_reissued_at":"2026-07-05T09:18:30.215415Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:18:30.215415Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2409.15934","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:18:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"G/zmMuk4aZXjX2okxonAMhSKN8W5RItir0WNhhbtFKnr1c9cSLAYKgex6pOhqt3lDbQ/H/2IridfFbWkANB7Bg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T09:25:13.428175Z"},"content_sha256":"84c8fb2aa9abe3b8da669a99e46c95c76ae833cbaaff8cc23a56225dce7a0fd3","schema_version":"1.0","event_id":"sha256:84c8fb2aa9abe3b8da669a99e46c95c76ae833cbaaff8cc23a56225dce7a0fd3"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:BQND6NDDR4MBD6M6VZRKULHREI","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Automated test generation to evaluate tool-augmented LLMs as conversational AI agents","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"David Aparicio, Mariana Almeida, Samuel Arcadinho","submitted_at":"2024-09-24T09:57:43Z","abstract_excerpt":"Tool-augmented LLMs are a promising approach to create AI agents that can have realistic conversations, follow procedures, and call appropriate functions. However, evaluating them is challenging due to the diversity of possible conversations, and existing datasets focus only on single interactions and function-calling. We present a test generation pipeline to evaluate LLMs as conversational AI agents. Our framework uses LLMs to generate diverse tests grounded on user-defined procedures. For that, we use intermediate graphs to limit the LLM test generator's tendency to hallucinate content that "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.15934","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2409.15934/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:18:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"6G57No4miu37MM1LYP7og2rsHj+Uw/3nyV95b2RflzdgVjiHR9dfISn2kaL1snN4DVa6TuQeutGNbR2FyMCFAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T09:25:13.428696Z"},"content_sha256":"59e093fe82f18adc3ce71857cbf0cd9be8035777eec2ffe0a1b83751ddae0027","schema_version":"1.0","event_id":"sha256:59e093fe82f18adc3ce71857cbf0cd9be8035777eec2ffe0a1b83751ddae0027"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/BQND6NDDR4MBD6M6VZRKULHREI/bundle.json","state_url":"https://pith.science/pith/BQND6NDDR4MBD6M6VZRKULHREI/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/BQND6NDDR4MBD6M6VZRKULHREI/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-11T09:25:13Z","links":{"resolver":"https://pith.science/pith/BQND6NDDR4MBD6M6VZRKULHREI","bundle":"https://pith.science/pith/BQND6NDDR4MBD6M6VZRKULHREI/bundle.json","state":"https://pith.science/pith/BQND6NDDR4MBD6M6VZRKULHREI/state.json","well_known_bundle":"https://pith.science/.well-known/pith/BQND6NDDR4MBD6M6VZRKULHREI/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:BQND6NDDR4MBD6M6VZRKULHREI","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"107db695b677e2f35b3b2b34b3776ed4a4e87d33e35d2107ab51e1cc998ba46c","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-24T09:57:43Z","title_canon_sha256":"775c3e9a3e969d79aa00a7733ff74288e677cf0a56d4b5eee1ea6ac82804acce"},"schema_version":"1.0","source":{"id":"2409.15934","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.15934","created_at":"2026-07-05T09:18:30Z"},{"alias_kind":"arxiv_version","alias_value":"2409.15934v2","created_at":"2026-07-05T09:18:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.15934","created_at":"2026-07-05T09:18:30Z"},{"alias_kind":"pith_short_12","alias_value":"BQND6NDDR4MB","created_at":"2026-07-05T09:18:30Z"},{"alias_kind":"pith_short_16","alias_value":"BQND6NDDR4MBD6M6","created_at":"2026-07-05T09:18:30Z"},{"alias_kind":"pith_short_8","alias_value":"BQND6NDD","created_at":"2026-07-05T09:18:30Z"}],"graph_snapshots":[{"event_id":"sha256:59e093fe82f18adc3ce71857cbf0cd9be8035777eec2ffe0a1b83751ddae0027","target":"graph","created_at":"2026-07-05T09:18:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2409.15934/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Tool-augmented LLMs are a promising approach to create AI agents that can have realistic conversations, follow procedures, and call appropriate functions. However, evaluating them is challenging due to the diversity of possible conversations, and existing datasets focus only on single interactions and function-calling. We present a test generation pipeline to evaluate LLMs as conversational AI agents. Our framework uses LLMs to generate diverse tests grounded on user-defined procedures. For that, we use intermediate graphs to limit the LLM test generator's tendency to hallucinate content that ","authors_text":"David Aparicio, Mariana Almeida, Samuel Arcadinho","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-24T09:57:43Z","title":"Automated test generation to evaluate tool-augmented LLMs as conversational AI agents"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.15934","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:84c8fb2aa9abe3b8da669a99e46c95c76ae833cbaaff8cc23a56225dce7a0fd3","target":"record","created_at":"2026-07-05T09:18:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"107db695b677e2f35b3b2b34b3776ed4a4e87d33e35d2107ab51e1cc998ba46c","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-24T09:57:43Z","title_canon_sha256":"775c3e9a3e969d79aa00a7733ff74288e677cf0a56d4b5eee1ea6ac82804acce"},"schema_version":"1.0","source":{"id":"2409.15934","kind":"arxiv","version":2}},"canonical_sha256":"0c1a3f34638f1811f99eae62aa2cf122199e6313d0f0cd489d97006091b399d6","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0c1a3f34638f1811f99eae62aa2cf122199e6313d0f0cd489d97006091b399d6","first_computed_at":"2026-07-05T09:18:30.215415Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:18:30.215415Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"BzjgJJ9SJahzBhpJuyUiSXlZcO+8BjJdls8hNL/dRFLC9Ja1I+eSrNIwnA3NTDZnwM7kevc1XQIonowYYrALDA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:18:30.215876Z","signed_message":"canonical_sha256_bytes"},"source_id":"2409.15934","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:84c8fb2aa9abe3b8da669a99e46c95c76ae833cbaaff8cc23a56225dce7a0fd3","sha256:59e093fe82f18adc3ce71857cbf0cd9be8035777eec2ffe0a1b83751ddae0027"],"state_sha256":"8a9daf197c8b838dbeca447d06929ba42900ce2202033d551efed173e3e49a0c"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"m/0tRdS1FSoL7KKxAuCzVyf3l9Q7a6/X+/aA/pAzlLqCypDGW+9Yd//UGAg3k6sNbKCJi1rcsz73LKQqZHekCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-11T09:25:13.433064Z","bundle_sha256":"32ee374d41b26819cc7c78d2366d4e684f963ad86aee13fe2238f23278b848da"}}