{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:F52V53DKGBUVML5SPCPDSAHSNC","short_pith_number":"pith:F52V53DK","canonical_record":{"source":{"id":"2407.10499","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-15T07:43:55Z","cross_cats_sorted":[],"title_canon_sha256":"d1dc0b4ba08f249bf092d1b6edd0d1e90035d1ffbf8c50f0b029fdd3c4705259","abstract_canon_sha256":"eaf30416a31d417bcea74474ba907a1869b2d2c26bdba72ec7e78df89afc2b9d"},"schema_version":"1.0"},"canonical_sha256":"2f755eec6a3069562fb2789e3900f268bd27d181a1de7fdb69225593b3725a15","source":{"kind":"arxiv","id":"2407.10499","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.10499","created_at":"2026-07-05T09:31:47Z"},{"alias_kind":"arxiv_version","alias_value":"2407.10499v3","created_at":"2026-07-05T09:31:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.10499","created_at":"2026-07-05T09:31:47Z"},{"alias_kind":"pith_short_12","alias_value":"F52V53DKGBUV","created_at":"2026-07-05T09:31:47Z"},{"alias_kind":"pith_short_16","alias_value":"F52V53DKGBUVML5S","created_at":"2026-07-05T09:31:47Z"},{"alias_kind":"pith_short_8","alias_value":"F52V53DK","created_at":"2026-07-05T09:31:47Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:F52V53DKGBUVML5SPCPDSAHSNC","target":"record","payload":{"canonical_record":{"source":{"id":"2407.10499","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-15T07:43:55Z","cross_cats_sorted":[],"title_canon_sha256":"d1dc0b4ba08f249bf092d1b6edd0d1e90035d1ffbf8c50f0b029fdd3c4705259","abstract_canon_sha256":"eaf30416a31d417bcea74474ba907a1869b2d2c26bdba72ec7e78df89afc2b9d"},"schema_version":"1.0"},"canonical_sha256":"2f755eec6a3069562fb2789e3900f268bd27d181a1de7fdb69225593b3725a15","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:31:47.470174Z","signature_b64":"erC05En8TxkXkJlbXKtp6tzmTZaTiibfXgNGpxlslhVIouTKNJfsyADrVeJIpvB92p29RHMoxO5J1D/SAh3jAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2f755eec6a3069562fb2789e3900f268bd27d181a1de7fdb69225593b3725a15","last_reissued_at":"2026-07-05T09:31:47.469670Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:31:47.469670Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2407.10499","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:31:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"kmlQyjx1pdx/0NzkgkSlIECQOKp0XnG2RBIirMG7SgpvMAR6wpN7GegwIDLBhImhgKWEnNiOPPHkzbrHjuUwCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-19T11:04:49.414546Z"},"content_sha256":"781f90057d86331fd8543f286d6113972ffeccc594cfd38f654e015fb9959f4f","schema_version":"1.0","event_id":"sha256:781f90057d86331fd8543f286d6113972ffeccc594cfd38f654e015fb9959f4f"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:F52V53DKGBUVML5SPCPDSAHSNC","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"CIBench: Evaluating Your LLMs with a Code Interpreter Plugin","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Chuyu Zhang, Dahua Lin, Fengzhe Zhou, Haowen Shen, Kai Chen, Kuikun Liu, Songyang Zhang, Wenwei Zhang, Xuming He, Yingfan Hu, Zerun Ma","submitted_at":"2024-07-15T07:43:55Z","abstract_excerpt":"While LLM-Based agents, which use external tools to solve complex problems, have made significant progress, benchmarking their ability is challenging, thereby hindering a clear understanding of their limitations. In this paper, we propose an interactive evaluation framework, named CIBench, to comprehensively assess LLMs' ability to utilize code interpreters for data science tasks. Our evaluation framework includes an evaluation dataset and two evaluation modes. The evaluation dataset is constructed using an LLM-human cooperative approach and simulates an authentic workflow by leveraging consec"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.10499","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2407.10499/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:31:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"1p+Y1eFzrLUpwz32zs06ilW/hFnY11mEcN5CiDVL3Kj86t8HT7uUNe6kfuLYp0KTuaC3Im/Mu17v21RE7G5jAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-19T11:04:49.416093Z"},"content_sha256":"89765490c348b23d1df8a185341f0fbb39ed02f1c7eb4e0461ee5b04cad5d1d1","schema_version":"1.0","event_id":"sha256:89765490c348b23d1df8a185341f0fbb39ed02f1c7eb4e0461ee5b04cad5d1d1"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/F52V53DKGBUVML5SPCPDSAHSNC/bundle.json","state_url":"https://pith.science/pith/F52V53DKGBUVML5SPCPDSAHSNC/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/F52V53DKGBUVML5SPCPDSAHSNC/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-19T11:04:49Z","links":{"resolver":"https://pith.science/pith/F52V53DKGBUVML5SPCPDSAHSNC","bundle":"https://pith.science/pith/F52V53DKGBUVML5SPCPDSAHSNC/bundle.json","state":"https://pith.science/pith/F52V53DKGBUVML5SPCPDSAHSNC/state.json","well_known_bundle":"https://pith.science/.well-known/pith/F52V53DKGBUVML5SPCPDSAHSNC/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:F52V53DKGBUVML5SPCPDSAHSNC","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"eaf30416a31d417bcea74474ba907a1869b2d2c26bdba72ec7e78df89afc2b9d","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-15T07:43:55Z","title_canon_sha256":"d1dc0b4ba08f249bf092d1b6edd0d1e90035d1ffbf8c50f0b029fdd3c4705259"},"schema_version":"1.0","source":{"id":"2407.10499","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.10499","created_at":"2026-07-05T09:31:47Z"},{"alias_kind":"arxiv_version","alias_value":"2407.10499v3","created_at":"2026-07-05T09:31:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.10499","created_at":"2026-07-05T09:31:47Z"},{"alias_kind":"pith_short_12","alias_value":"F52V53DKGBUV","created_at":"2026-07-05T09:31:47Z"},{"alias_kind":"pith_short_16","alias_value":"F52V53DKGBUVML5S","created_at":"2026-07-05T09:31:47Z"},{"alias_kind":"pith_short_8","alias_value":"F52V53DK","created_at":"2026-07-05T09:31:47Z"}],"graph_snapshots":[{"event_id":"sha256:89765490c348b23d1df8a185341f0fbb39ed02f1c7eb4e0461ee5b04cad5d1d1","target":"graph","created_at":"2026-07-05T09:31:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2407.10499/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"While LLM-Based agents, which use external tools to solve complex problems, have made significant progress, benchmarking their ability is challenging, thereby hindering a clear understanding of their limitations. In this paper, we propose an interactive evaluation framework, named CIBench, to comprehensively assess LLMs' ability to utilize code interpreters for data science tasks. Our evaluation framework includes an evaluation dataset and two evaluation modes. The evaluation dataset is constructed using an LLM-human cooperative approach and simulates an authentic workflow by leveraging consec","authors_text":"Chuyu Zhang, Dahua Lin, Fengzhe Zhou, Haowen Shen, Kai Chen, Kuikun Liu, Songyang Zhang, Wenwei Zhang, Xuming He, Yingfan Hu, Zerun Ma","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-15T07:43:55Z","title":"CIBench: Evaluating Your LLMs with a Code Interpreter Plugin"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.10499","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:781f90057d86331fd8543f286d6113972ffeccc594cfd38f654e015fb9959f4f","target":"record","created_at":"2026-07-05T09:31:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"eaf30416a31d417bcea74474ba907a1869b2d2c26bdba72ec7e78df89afc2b9d","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-15T07:43:55Z","title_canon_sha256":"d1dc0b4ba08f249bf092d1b6edd0d1e90035d1ffbf8c50f0b029fdd3c4705259"},"schema_version":"1.0","source":{"id":"2407.10499","kind":"arxiv","version":3}},"canonical_sha256":"2f755eec6a3069562fb2789e3900f268bd27d181a1de7fdb69225593b3725a15","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"2f755eec6a3069562fb2789e3900f268bd27d181a1de7fdb69225593b3725a15","first_computed_at":"2026-07-05T09:31:47.469670Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:31:47.469670Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"erC05En8TxkXkJlbXKtp6tzmTZaTiibfXgNGpxlslhVIouTKNJfsyADrVeJIpvB92p29RHMoxO5J1D/SAh3jAg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:31:47.470174Z","signed_message":"canonical_sha256_bytes"},"source_id":"2407.10499","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:781f90057d86331fd8543f286d6113972ffeccc594cfd38f654e015fb9959f4f","sha256:89765490c348b23d1df8a185341f0fbb39ed02f1c7eb4e0461ee5b04cad5d1d1"],"state_sha256":"93deca61ca30118d4ae7bd24bf187797f670a5308c0e1d6fb7574e2b57a5c715"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"omC/ruDknum/xbo3dzNBy/DqSwNunVa/ruMLShhaxP4m5onnZ2MKY1XZraCScp5+XyklTKc6HfD9/c0bR2rNBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-19T11:04:49.424019Z","bundle_sha256":"310b092cd8033cf3844d1527c595f57f5defdb6d4f57e4849d8ed4b8e54b5530"}}