{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:FZQKG5NC4HPZN7AOSEQ7CMQTML","short_pith_number":"pith:FZQKG5NC","canonical_record":{"source":{"id":"2404.06411","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-04-09T16:01:24Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"d1d8ed4fc75414ea3a9e69947d6b7eaa1e596f153544305ebd0b57e53d2c1138","abstract_canon_sha256":"bd10170c8f130187734c8934d6cc15604d5bd34ab7ea4e3cb1874e9f44c62cfe"},"schema_version":"1.0"},"canonical_sha256":"2e60a375a2e1df96fc0e9121f1321362da20af5cf0dd61eeed6d78f082f71063","source":{"kind":"arxiv","id":"2404.06411","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.06411","created_at":"2026-07-05T08:06:08Z"},{"alias_kind":"arxiv_version","alias_value":"2404.06411v1","created_at":"2026-07-05T08:06:08Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.06411","created_at":"2026-07-05T08:06:08Z"},{"alias_kind":"pith_short_12","alias_value":"FZQKG5NC4HPZ","created_at":"2026-07-05T08:06:08Z"},{"alias_kind":"pith_short_16","alias_value":"FZQKG5NC4HPZN7AO","created_at":"2026-07-05T08:06:08Z"},{"alias_kind":"pith_short_8","alias_value":"FZQKG5NC","created_at":"2026-07-05T08:06:08Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:FZQKG5NC4HPZN7AOSEQ7CMQTML","target":"record","payload":{"canonical_record":{"source":{"id":"2404.06411","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-04-09T16:01:24Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"d1d8ed4fc75414ea3a9e69947d6b7eaa1e596f153544305ebd0b57e53d2c1138","abstract_canon_sha256":"bd10170c8f130187734c8934d6cc15604d5bd34ab7ea4e3cb1874e9f44c62cfe"},"schema_version":"1.0"},"canonical_sha256":"2e60a375a2e1df96fc0e9121f1321362da20af5cf0dd61eeed6d78f082f71063","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:06:08.705305Z","signature_b64":"yW9cxftA6McmAed32n4a0sxXrxLKXZsNbmbytuwTiZYGVtegUK4bXpbhtzqzXh5x4q7HFqHA1GvKSutn2EilDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2e60a375a2e1df96fc0e9121f1321362da20af5cf0dd61eeed6d78f082f71063","last_reissued_at":"2026-07-05T08:06:08.704881Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:06:08.704881Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2404.06411","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:06:08Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"TqXa4Vlb4t7Re6t6XyRg/z3cPZ3BHwACXRdjT4HewZaiIWepScAiSBxKr5d26t0gAjTzXCTZsVyPCPT0FgvqAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T23:23:40.495399Z"},"content_sha256":"71594c0761ece3045ff501371f3d7358672e19c351309298359cc1f69968f033","schema_version":"1.0","event_id":"sha256:71594c0761ece3045ff501371f3d7358672e19c351309298359cc1f69968f033"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:FZQKG5NC4HPZN7AOSEQ7CMQTML","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"AgentQuest: A Modular Benchmark Framework to Measure Progress and Improve LLM Agents","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.AI","authors_text":"Carolin Lawrence, Davide Sanvito, David Friede, Giuseppe Siracusano, Kiril Gashteovski, Luca Gioacchini, Roberto Bifulco","submitted_at":"2024-04-09T16:01:24Z","abstract_excerpt":"The advances made by Large Language Models (LLMs) have led to the pursuit of LLM agents that can solve intricate, multi-step reasoning tasks. As with any research pursuit, benchmarking and evaluation are key corner stones to efficient and reliable progress. However, existing benchmarks are often narrow and simply compute overall task success. To face these issues, we propose AgentQuest -- a framework where (i) both benchmarks and metrics are modular and easily extensible through well documented and easy-to-use APIs; (ii) we offer two new evaluation metrics that can reliably track LLM agent pro"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.06411","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2404.06411/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:06:08Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"2pGiDz7cZ2+g7nZMDhEXapXQpPregKY2XrKVIlRMUb/xikToy/L0No8BJ/ywG7YMTc8+2bz60l0yQ5swT/wtDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-03T23:23:40.495907Z"},"content_sha256":"2272b3b0ceacd2b8b63c5e3c444a3da7f3805a69c8468603d5d841e6195d5e44","schema_version":"1.0","event_id":"sha256:2272b3b0ceacd2b8b63c5e3c444a3da7f3805a69c8468603d5d841e6195d5e44"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/FZQKG5NC4HPZN7AOSEQ7CMQTML/bundle.json","state_url":"https://pith.science/pith/FZQKG5NC4HPZN7AOSEQ7CMQTML/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/FZQKG5NC4HPZN7AOSEQ7CMQTML/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-03T23:23:40Z","links":{"resolver":"https://pith.science/pith/FZQKG5NC4HPZN7AOSEQ7CMQTML","bundle":"https://pith.science/pith/FZQKG5NC4HPZN7AOSEQ7CMQTML/bundle.json","state":"https://pith.science/pith/FZQKG5NC4HPZN7AOSEQ7CMQTML/state.json","well_known_bundle":"https://pith.science/.well-known/pith/FZQKG5NC4HPZN7AOSEQ7CMQTML/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:FZQKG5NC4HPZN7AOSEQ7CMQTML","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"bd10170c8f130187734c8934d6cc15604d5bd34ab7ea4e3cb1874e9f44c62cfe","cross_cats_sorted":["cs.CL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-04-09T16:01:24Z","title_canon_sha256":"d1d8ed4fc75414ea3a9e69947d6b7eaa1e596f153544305ebd0b57e53d2c1138"},"schema_version":"1.0","source":{"id":"2404.06411","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.06411","created_at":"2026-07-05T08:06:08Z"},{"alias_kind":"arxiv_version","alias_value":"2404.06411v1","created_at":"2026-07-05T08:06:08Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.06411","created_at":"2026-07-05T08:06:08Z"},{"alias_kind":"pith_short_12","alias_value":"FZQKG5NC4HPZ","created_at":"2026-07-05T08:06:08Z"},{"alias_kind":"pith_short_16","alias_value":"FZQKG5NC4HPZN7AO","created_at":"2026-07-05T08:06:08Z"},{"alias_kind":"pith_short_8","alias_value":"FZQKG5NC","created_at":"2026-07-05T08:06:08Z"}],"graph_snapshots":[{"event_id":"sha256:2272b3b0ceacd2b8b63c5e3c444a3da7f3805a69c8468603d5d841e6195d5e44","target":"graph","created_at":"2026-07-05T08:06:08Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2404.06411/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The advances made by Large Language Models (LLMs) have led to the pursuit of LLM agents that can solve intricate, multi-step reasoning tasks. As with any research pursuit, benchmarking and evaluation are key corner stones to efficient and reliable progress. However, existing benchmarks are often narrow and simply compute overall task success. To face these issues, we propose AgentQuest -- a framework where (i) both benchmarks and metrics are modular and easily extensible through well documented and easy-to-use APIs; (ii) we offer two new evaluation metrics that can reliably track LLM agent pro","authors_text":"Carolin Lawrence, Davide Sanvito, David Friede, Giuseppe Siracusano, Kiril Gashteovski, Luca Gioacchini, Roberto Bifulco","cross_cats":["cs.CL"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-04-09T16:01:24Z","title":"AgentQuest: A Modular Benchmark Framework to Measure Progress and Improve LLM Agents"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.06411","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:71594c0761ece3045ff501371f3d7358672e19c351309298359cc1f69968f033","target":"record","created_at":"2026-07-05T08:06:08Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"bd10170c8f130187734c8934d6cc15604d5bd34ab7ea4e3cb1874e9f44c62cfe","cross_cats_sorted":["cs.CL"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-04-09T16:01:24Z","title_canon_sha256":"d1d8ed4fc75414ea3a9e69947d6b7eaa1e596f153544305ebd0b57e53d2c1138"},"schema_version":"1.0","source":{"id":"2404.06411","kind":"arxiv","version":1}},"canonical_sha256":"2e60a375a2e1df96fc0e9121f1321362da20af5cf0dd61eeed6d78f082f71063","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"2e60a375a2e1df96fc0e9121f1321362da20af5cf0dd61eeed6d78f082f71063","first_computed_at":"2026-07-05T08:06:08.704881Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:06:08.704881Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"yW9cxftA6McmAed32n4a0sxXrxLKXZsNbmbytuwTiZYGVtegUK4bXpbhtzqzXh5x4q7HFqHA1GvKSutn2EilDg==","signature_status":"signed_v1","signed_at":"2026-07-05T08:06:08.705305Z","signed_message":"canonical_sha256_bytes"},"source_id":"2404.06411","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:71594c0761ece3045ff501371f3d7358672e19c351309298359cc1f69968f033","sha256:2272b3b0ceacd2b8b63c5e3c444a3da7f3805a69c8468603d5d841e6195d5e44"],"state_sha256":"fedca911bb25495ebaa350e8edac6736cfa921315df866ff3e7fa4b940d60b34"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"WU1nJjMgbRueI2qZtnSSMBIjsAXeoU2WFB0HAUjkpIhAKMY+pIBXETLQRvqvbk4bn+lfVmnN2K838VunN1qwAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-03T23:23:40.499752Z","bundle_sha256":"a1d299a522334ab574c8d5c8af00153704e464468eee67ad6280073bcc9a789a"}}