{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2026:CZZMY6FBQHAXED3CZOMW5EWPGP","short_pith_number":"pith:CZZMY6FB","canonical_record":{"source":{"id":"2605.14133","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2026-05-13T21:34:08Z","cross_cats_sorted":[],"title_canon_sha256":"ce555f5f97c1a6fef39238bcef7a08a72616148e4e6f90f86baf4d72c056491d","abstract_canon_sha256":"cc686c0882d6287dccdf6f47c8720a1a11370f9ebf93297dbfb3c92e8f376d7e"},"schema_version":"1.0"},"canonical_sha256":"1672cc78a181c1720f62cb996e92cf33de18527add55b7b3b71745a47fc47ded","source":{"kind":"arxiv","id":"2605.14133","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2605.14133","created_at":"2026-05-17T23:39:11Z"},{"alias_kind":"arxiv_version","alias_value":"2605.14133v1","created_at":"2026-05-17T23:39:11Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2605.14133","created_at":"2026-05-17T23:39:11Z"},{"alias_kind":"pith_short_12","alias_value":"CZZMY6FBQHAX","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_16","alias_value":"CZZMY6FBQHAXED3C","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_8","alias_value":"CZZMY6FB","created_at":"2026-05-18T12:33:37Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2026:CZZMY6FBQHAXED3CZOMW5EWPGP","target":"record","payload":{"canonical_record":{"source":{"id":"2605.14133","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2026-05-13T21:34:08Z","cross_cats_sorted":[],"title_canon_sha256":"ce555f5f97c1a6fef39238bcef7a08a72616148e4e6f90f86baf4d72c056491d","abstract_canon_sha256":"cc686c0882d6287dccdf6f47c8720a1a11370f9ebf93297dbfb3c92e8f376d7e"},"schema_version":"1.0"},"canonical_sha256":"1672cc78a181c1720f62cb996e92cf33de18527add55b7b3b71745a47fc47ded","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:39:11.771807Z","signature_b64":"1TtCftNXWYZFdx+SHJBfQmxYZM5yrUSg/kjuG1Of5b2Ptm98Xsjpa4lpf4/W4Zj2Feg6G7ygozCvndTA44bcCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1672cc78a181c1720f62cb996e92cf33de18527add55b7b3b71745a47fc47ded","last_reissued_at":"2026-05-17T23:39:11.771097Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:39:11.771097Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2605.14133","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:39:11Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"f0Ob33PuTHPnvgOY/bWOL2NkcLTNDlxWmu3tySPYv89DBZnFh8vIOOm0FFwz09IuKWxMWjPTEcfOMh36Kp2iBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-27T10:42:37.466419Z"},"content_sha256":"1912d8efb6192752d09b0b22a2bb3ae70aefde1d8728c4650ee71074443295a2","schema_version":"1.0","event_id":"sha256:1912d8efb6192752d09b0b22a2bb3ae70aefde1d8728c4650ee71074443295a2"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2026:CZZMY6FBQHAXED3CZOMW5EWPGP","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"ClawForge: Generating Executable Interactive Benchmarks for Command-Line Agents","license":"http://creativecommons.org/licenses/by/4.0/","headline":"ClawForge generates executable command-line benchmarks that test agents on pre-existing state conflicts, with top models reaching only 45.3 percent strict accuracy.","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Cihang Xie, Fang Wu, Haonian Ji, Huaxiu Yao, Jiaqi Liu, Jike Zhong, Kaide Zeng, Kaiwen Xiong, Peng Xia, Yuxiang Lai, Zeyu Zheng","submitted_at":"2026-05-13T21:34:08Z","abstract_excerpt":"Interactive agent benchmarks face a tension between scalable construction and realistic workflow evaluation. Hand-authored tasks are expensive to extend and revise, while static prompt evaluation misses failures that only appear when agents operate over persistent state. Existing interactive benchmarks have advanced agent evaluation significantly, but most initialize tasks from clean state and do not systematically test how agents handle pre-existing partial, stale, or conflicting artifacts. We present \\textbf{ClawForge}, a generator-backed benchmark framework for executable command-line workf"},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"The best model reaches only 45.3% strict accuracy, wrong-state replacement remains below 17% for all models, and the widest model separation (17% to 90%) is driven by whether agents inspect existing state before acting.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"That the 17 generated scenarios and their validators faithfully capture the distribution of state conflicts that appear in real command-line workflows.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"ClawForge supplies a generator that turns scenario templates into reproducible command-line tasks testing state conflict handling, where the strongest frontier model scores only 45.3 percent strict accuracy.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"ClawForge generates executable command-line benchmarks that test agents on pre-existing state conflicts, with top models reaching only 45.3 percent strict accuracy.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"2ee5d2bd95cbc87dc553c84180e5b14f5df1a8f260323bd3472006ae9a3a8a5c"},"source":{"id":"2605.14133","kind":"arxiv","version":1},"verdict":{"id":"3a189ea1-c824-4180-9338-625bdf6ee19f","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-15T05:04:49.987617Z","strongest_claim":"The best model reaches only 45.3% strict accuracy, wrong-state replacement remains below 17% for all models, and the widest model separation (17% to 90%) is driven by whether agents inspect existing state before acting.","one_line_summary":"ClawForge supplies a generator that turns scenario templates into reproducible command-line tasks testing state conflict handling, where the strongest frontier model scores only 45.3 percent strict accuracy.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"That the 17 generated scenarios and their validators faithfully capture the distribution of state conflicts that appear in real command-line workflows.","pith_extraction_headline":"ClawForge generates executable command-line benchmarks that test agents on pre-existing state conflicts, with top models reaching only 45.3 percent strict accuracy."},"references":{"count":101,"sample":[{"doi":"","year":null,"title":"A-MEM: Agentic Memory for LLM Agents","work_id":"3b98feb2-fdb1-479a-bbe4-2c298a4592e2","ref_index":1,"cited_arxiv_id":"2502.12110","is_internal_anchor":true},{"doi":"","year":null,"title":"Zep: A Temporal Knowledge Graph Architecture for Agent Memory","work_id":"515c933e-12ae-439d-a7ff-c07fee482dfb","ref_index":2,"cited_arxiv_id":"2501.13956","is_internal_anchor":true},{"doi":"","year":null,"title":"Proceedings of the AAAI conference on artificial intelligence , volume=","work_id":"a60807ef-dd92-4b41-94d6-cd81a67ea2ec","ref_index":3,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2025,"title":"Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing , pages=","work_id":"03eb5914-f16b-41a5-a3eb-1ec8872212cc","ref_index":4,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2023,"title":"MemGPT: towards LLMs as operating systems. , author=. 2023 , publisher=","work_id":"28a11e01-274f-4df2-ba22-74b5dc701483","ref_index":5,"cited_arxiv_id":"","is_internal_anchor":false}],"resolved_work":101,"snapshot_sha256":"73f03d667bb20d289ff6e64cd17e7f66b5f847f3d88df5533f47587b3e6cc146","internal_anchors":42},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":"3a189ea1-c824-4180-9338-625bdf6ee19f"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:39:11Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"k6pWEnpwnOA9EEzHU0QdEFXOemhdHlzNDfq8iXP/DPndtc9sXdoznR4T9z1GSZfg8mBR78Djv3e71dzYV9vvDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-27T10:42:37.467517Z"},"content_sha256":"8bd61fbbff2604fa792ee0d7a10ddce9157aab95b2542fa73917b710e9f9ecb4","schema_version":"1.0","event_id":"sha256:8bd61fbbff2604fa792ee0d7a10ddce9157aab95b2542fa73917b710e9f9ecb4"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/CZZMY6FBQHAXED3CZOMW5EWPGP/bundle.json","state_url":"https://pith.science/pith/CZZMY6FBQHAXED3CZOMW5EWPGP/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/CZZMY6FBQHAXED3CZOMW5EWPGP/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-05-27T10:42:37Z","links":{"resolver":"https://pith.science/pith/CZZMY6FBQHAXED3CZOMW5EWPGP","bundle":"https://pith.science/pith/CZZMY6FBQHAXED3CZOMW5EWPGP/bundle.json","state":"https://pith.science/pith/CZZMY6FBQHAXED3CZOMW5EWPGP/state.json","well_known_bundle":"https://pith.science/.well-known/pith/CZZMY6FBQHAXED3CZOMW5EWPGP/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2026:CZZMY6FBQHAXED3CZOMW5EWPGP","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"cc686c0882d6287dccdf6f47c8720a1a11370f9ebf93297dbfb3c92e8f376d7e","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2026-05-13T21:34:08Z","title_canon_sha256":"ce555f5f97c1a6fef39238bcef7a08a72616148e4e6f90f86baf4d72c056491d"},"schema_version":"1.0","source":{"id":"2605.14133","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2605.14133","created_at":"2026-05-17T23:39:11Z"},{"alias_kind":"arxiv_version","alias_value":"2605.14133v1","created_at":"2026-05-17T23:39:11Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2605.14133","created_at":"2026-05-17T23:39:11Z"},{"alias_kind":"pith_short_12","alias_value":"CZZMY6FBQHAX","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_16","alias_value":"CZZMY6FBQHAXED3C","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_8","alias_value":"CZZMY6FB","created_at":"2026-05-18T12:33:37Z"}],"graph_snapshots":[{"event_id":"sha256:8bd61fbbff2604fa792ee0d7a10ddce9157aab95b2542fa73917b710e9f9ecb4","target":"graph","created_at":"2026-05-17T23:39:11Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"The best model reaches only 45.3% strict accuracy, wrong-state replacement remains below 17% for all models, and the widest model separation (17% to 90%) is driven by whether agents inspect existing state before acting."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"That the 17 generated scenarios and their validators faithfully capture the distribution of state conflicts that appear in real command-line workflows."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"ClawForge supplies a generator that turns scenario templates into reproducible command-line tasks testing state conflict handling, where the strongest frontier model scores only 45.3 percent strict accuracy."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"ClawForge generates executable command-line benchmarks that test agents on pre-existing state conflicts, with top models reaching only 45.3 percent strict accuracy."}],"snapshot_sha256":"2ee5d2bd95cbc87dc553c84180e5b14f5df1a8f260323bd3472006ae9a3a8a5c"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"Interactive agent benchmarks face a tension between scalable construction and realistic workflow evaluation. Hand-authored tasks are expensive to extend and revise, while static prompt evaluation misses failures that only appear when agents operate over persistent state. Existing interactive benchmarks have advanced agent evaluation significantly, but most initialize tasks from clean state and do not systematically test how agents handle pre-existing partial, stale, or conflicting artifacts. We present \\textbf{ClawForge}, a generator-backed benchmark framework for executable command-line workf","authors_text":"Cihang Xie, Fang Wu, Haonian Ji, Huaxiu Yao, Jiaqi Liu, Jike Zhong, Kaide Zeng, Kaiwen Xiong, Peng Xia, Yuxiang Lai, Zeyu Zheng","cross_cats":[],"headline":"ClawForge generates executable command-line benchmarks that test agents on pre-existing state conflicts, with top models reaching only 45.3 percent strict accuracy.","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2026-05-13T21:34:08Z","title":"ClawForge: Generating Executable Interactive Benchmarks for Command-Line Agents"},"references":{"count":101,"internal_anchors":42,"resolved_work":101,"sample":[{"cited_arxiv_id":"2502.12110","doi":"","is_internal_anchor":true,"ref_index":1,"title":"A-MEM: Agentic Memory for LLM Agents","work_id":"3b98feb2-fdb1-479a-bbe4-2c298a4592e2","year":null},{"cited_arxiv_id":"2501.13956","doi":"","is_internal_anchor":true,"ref_index":2,"title":"Zep: A Temporal Knowledge Graph Architecture for Agent Memory","work_id":"515c933e-12ae-439d-a7ff-c07fee482dfb","year":null},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":3,"title":"Proceedings of the AAAI conference on artificial intelligence , volume=","work_id":"a60807ef-dd92-4b41-94d6-cd81a67ea2ec","year":null},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":4,"title":"Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing , pages=","work_id":"03eb5914-f16b-41a5-a3eb-1ec8872212cc","year":2025},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":5,"title":"MemGPT: towards LLMs as operating systems. , author=. 2023 , publisher=","work_id":"28a11e01-274f-4df2-ba22-74b5dc701483","year":2023}],"snapshot_sha256":"73f03d667bb20d289ff6e64cd17e7f66b5f847f3d88df5533f47587b3e6cc146"},"source":{"id":"2605.14133","kind":"arxiv","version":1},"verdict":{"created_at":"2026-05-15T05:04:49.987617Z","id":"3a189ea1-c824-4180-9338-625bdf6ee19f","model_set":{"reader":"grok-4.3"},"one_line_summary":"ClawForge supplies a generator that turns scenario templates into reproducible command-line tasks testing state conflict handling, where the strongest frontier model scores only 45.3 percent strict accuracy.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"ClawForge generates executable command-line benchmarks that test agents on pre-existing state conflicts, with top models reaching only 45.3 percent strict accuracy.","strongest_claim":"The best model reaches only 45.3% strict accuracy, wrong-state replacement remains below 17% for all models, and the widest model separation (17% to 90%) is driven by whether agents inspect existing state before acting.","weakest_assumption":"That the 17 generated scenarios and their validators faithfully capture the distribution of state conflicts that appear in real command-line workflows."}},"verdict_id":"3a189ea1-c824-4180-9338-625bdf6ee19f"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1912d8efb6192752d09b0b22a2bb3ae70aefde1d8728c4650ee71074443295a2","target":"record","created_at":"2026-05-17T23:39:11Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"cc686c0882d6287dccdf6f47c8720a1a11370f9ebf93297dbfb3c92e8f376d7e","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2026-05-13T21:34:08Z","title_canon_sha256":"ce555f5f97c1a6fef39238bcef7a08a72616148e4e6f90f86baf4d72c056491d"},"schema_version":"1.0","source":{"id":"2605.14133","kind":"arxiv","version":1}},"canonical_sha256":"1672cc78a181c1720f62cb996e92cf33de18527add55b7b3b71745a47fc47ded","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"1672cc78a181c1720f62cb996e92cf33de18527add55b7b3b71745a47fc47ded","first_computed_at":"2026-05-17T23:39:11.771097Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:39:11.771097Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"1TtCftNXWYZFdx+SHJBfQmxYZM5yrUSg/kjuG1Of5b2Ptm98Xsjpa4lpf4/W4Zj2Feg6G7ygozCvndTA44bcCQ==","signature_status":"signed_v1","signed_at":"2026-05-17T23:39:11.771807Z","signed_message":"canonical_sha256_bytes"},"source_id":"2605.14133","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1912d8efb6192752d09b0b22a2bb3ae70aefde1d8728c4650ee71074443295a2","sha256:8bd61fbbff2604fa792ee0d7a10ddce9157aab95b2542fa73917b710e9f9ecb4"],"state_sha256":"7f20ecf14723b199a77f19d56342019375835abc950b0324ce2e007e1f212d2d"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"XYJyk6ZrPMf0OdkJFX+JYUfyDS6xbr/tknanrH6Z1rdBtZAmw+/yi1Dhe5hLrF+SGKsymcEGJGyf94KTyM3PCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-05-27T10:42:37.472305Z","bundle_sha256":"191e1d5dd5c53d378318576645ab7ced34d3cb61b1a1d688a1b42fc5b6e87eba"}}