{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:3AKINPDPEY5E4DLPOLPFFZDOXI","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"fdacaba0a601f052017bef4adecc766750331cb6d24f763c7c1e3ddc86303bda","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-04-28T11:08:22Z","title_canon_sha256":"4dfa9e24fcc765bf15db0d4e228dd847b5d56f8dd817458a312a013cd5948841"},"schema_version":"1.0","source":{"id":"2504.19678","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2504.19678","created_at":"2026-05-17T23:38:53Z"},{"alias_kind":"arxiv_version","alias_value":"2504.19678v2","created_at":"2026-05-17T23:38:53Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.19678","created_at":"2026-05-17T23:38:53Z"},{"alias_kind":"pith_short_12","alias_value":"3AKINPDPEY5E","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_16","alias_value":"3AKINPDPEY5E4DLP","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_8","alias_value":"3AKINPDP","created_at":"2026-05-18T12:33:37Z"}],"graph_snapshots":[{"event_id":"sha256:b70c51082beb5806001bf11d4499f7a9a6b14ba7bb8fc65a55ee4a8ac5ab4acc","target":"graph","created_at":"2026-05-17T23:38:53Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"we present a side-by-side comparison of benchmarks developed between 2019 and 2025 that evaluate these models and agents across multiple domains... we propose a taxonomy of approximately 60 benchmarks that cover general and academic knowledge reasoning, mathematical problem-solving, code generation and software engineering, factual grounding and retrieval, domain-specific evaluations, multimodal and embodied tasks, task orchestration, and interactive assessments."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"the landscape remains fragmented and lacks a unified taxonomy or comprehensive survey, which the authors' proposed taxonomy of approximately 60 benchmarks is assumed to resolve without major omissions or selection bias in the covered works."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"A survey consolidating benchmarks, agent frameworks, real-world applications, and protocols for LLM-based autonomous agents into a proposed taxonomy with recommendations for future research."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"A review organizes roughly 60 benchmarks for large language models and autonomous agents into one taxonomy covering reasoning, code, and real-world tasks."}],"snapshot_sha256":"12155fb9a0f94458fa89bacc37f0cce8fac32ae1941962551607bff5de458453"},"formal_canon":{"evidence_count":1,"snapshot_sha256":"df92f97a9cc1f0285d2d10bcc1fb6d1a80591f55a84b879debac6c25f0c3f401"},"paper":{"abstract_excerpt":"Large language models and autonomous AI agents have evolved rapidly, resulting in a diverse array of evaluation benchmarks, frameworks, and collaboration protocols. Driven by the growing need for standardized evaluation and integration, we systematically consolidate these fragmented efforts into a unified framework. However, the landscape remains fragmented and lacks a unified taxonomy or comprehensive survey. Therefore, we present a side-by-side comparison of benchmarks developed between 2019 and 2025 that evaluate these models and agents across multiple domains. In addition, we propose a tax","authors_text":"Merouane Debbah, Mohamed Amine Ferrag, Norbert Tihanyi","cross_cats":["cs.LG"],"headline":"A review organizes roughly 60 benchmarks for large language models and autonomous agents into one taxonomy covering reasoning, code, and real-world tasks.","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-04-28T11:08:22Z","title":"From LLM Reasoning to Autonomous AI Agents: A Comprehensive Review"},"references":{"count":236,"internal_anchors":43,"resolved_work":236,"sample":[{"cited_arxiv_id":"2412.16720","doi":"","is_internal_anchor":true,"ref_index":1,"title":"OpenAI o1 System Card","work_id":"68d3c334-0fc9-49e3-b7b0-a69afae933e2","year":2024},{"cited_arxiv_id":"2503.20215","doi":"","is_internal_anchor":true,"ref_index":2,"title":"Qwen2.5-Omni Technical Report","work_id":"438f105c-fa9b-44aa-ad52-43acb8045cda","year":2025},{"cited_arxiv_id":"2501.12948","doi":"","is_internal_anchor":true,"ref_index":3,"title":"DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning","work_id":"e6b75ad5-2877-4168-97c8-710407094d20","year":2025},{"cited_arxiv_id":"2407.21783","doi":"","is_internal_anchor":true,"ref_index":4,"title":"The Llama 3 Herd of Models","work_id":"1549a635-88af-4ac1-acfe-51ae7bb53345","year":2024},{"cited_arxiv_id":"2402.02716","doi":"","is_internal_anchor":true,"ref_index":5,"title":"Understanding the planning of LLM agents: A survey","work_id":"6daa5311-7f56-401d-94db-17be43e95cbc","year":2024}],"snapshot_sha256":"d9a8e56106c8959ad2eeb33339464e4cc83c02bd5819d7e3eb8b20a0074ce7b8"},"source":{"id":"2504.19678","kind":"arxiv","version":2},"verdict":{"created_at":"2026-05-15T02:53:18.046467Z","id":"e97de2dc-ed32-4712-8b7f-1ded3d7cb153","model_set":{"reader":"grok-4.3"},"one_line_summary":"A survey consolidating benchmarks, agent frameworks, real-world applications, and protocols for LLM-based autonomous agents into a proposed taxonomy with recommendations for future research.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"A review organizes roughly 60 benchmarks for large language models and autonomous agents into one taxonomy covering reasoning, code, and real-world tasks.","strongest_claim":"we present a side-by-side comparison of benchmarks developed between 2019 and 2025 that evaluate these models and agents across multiple domains... we propose a taxonomy of approximately 60 benchmarks that cover general and academic knowledge reasoning, mathematical problem-solving, code generation and software engineering, factual grounding and retrieval, domain-specific evaluations, multimodal and embodied tasks, task orchestration, and interactive assessments.","weakest_assumption":"the landscape remains fragmented and lacks a unified taxonomy or comprehensive survey, which the authors' proposed taxonomy of approximately 60 benchmarks is assumed to resolve without major omissions or selection bias in the covered works."}},"verdict_id":"e97de2dc-ed32-4712-8b7f-1ded3d7cb153"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:cfa926f1ef32e2f03a8ff5daafb1876eacf577004c9b4b2bde1a8bae22fc7e39","target":"record","created_at":"2026-05-17T23:38:53Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"fdacaba0a601f052017bef4adecc766750331cb6d24f763c7c1e3ddc86303bda","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-04-28T11:08:22Z","title_canon_sha256":"4dfa9e24fcc765bf15db0d4e228dd847b5d56f8dd817458a312a013cd5948841"},"schema_version":"1.0","source":{"id":"2504.19678","kind":"arxiv","version":2}},"canonical_sha256":"d81486bc6f263a4e0d6f72de52e46eba1eacab61c38fac9bd7279d6350bc4f6b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"d81486bc6f263a4e0d6f72de52e46eba1eacab61c38fac9bd7279d6350bc4f6b","first_computed_at":"2026-05-17T23:38:53.741627Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:38:53.741627Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"+huIwtPLopH0xkWG88XWhCdsv8D9/dKG1JDhgCA8cXXrWgF0y5GkAAtgEZu4XM+0bYlhV2wyEjsh75QvlNO9DQ==","signature_status":"signed_v1","signed_at":"2026-05-17T23:38:53.742358Z","signed_message":"canonical_sha256_bytes"},"source_id":"2504.19678","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:cfa926f1ef32e2f03a8ff5daafb1876eacf577004c9b4b2bde1a8bae22fc7e39","sha256:b70c51082beb5806001bf11d4499f7a9a6b14ba7bb8fc65a55ee4a8ac5ab4acc"],"state_sha256":"04bb8d19e2a9a4bf2e8e65126a18779f8676698223752d890a4a8841f11deba5"}