{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:YNDE57DAXA77KMEEM3QDZRNMKR","short_pith_number":"pith:YNDE57DA","canonical_record":{"source":{"id":"2309.10691","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-19T15:25:42Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"cf3f89404843380027e7ad4dd376f4cb4143bac52ed473efa7112f58e0f62aa4","abstract_canon_sha256":"96d0a49155bf6b4b61d6848710bb0274eba18da77a2359e2d550b724045b7e10"},"schema_version":"1.0"},"canonical_sha256":"c3464efc60b83ff5308466e03cc5ac547b41995dbdf14c473d2856826d4e9237","source":{"kind":"arxiv","id":"2309.10691","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.10691","created_at":"2026-07-05T07:55:13Z"},{"alias_kind":"arxiv_version","alias_value":"2309.10691v3","created_at":"2026-07-05T07:55:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.10691","created_at":"2026-07-05T07:55:13Z"},{"alias_kind":"pith_short_12","alias_value":"YNDE57DAXA77","created_at":"2026-07-05T07:55:13Z"},{"alias_kind":"pith_short_16","alias_value":"YNDE57DAXA77KMEE","created_at":"2026-07-05T07:55:13Z"},{"alias_kind":"pith_short_8","alias_value":"YNDE57DA","created_at":"2026-07-05T07:55:13Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:YNDE57DAXA77KMEEM3QDZRNMKR","target":"record","payload":{"canonical_record":{"source":{"id":"2309.10691","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-19T15:25:42Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"cf3f89404843380027e7ad4dd376f4cb4143bac52ed473efa7112f58e0f62aa4","abstract_canon_sha256":"96d0a49155bf6b4b61d6848710bb0274eba18da77a2359e2d550b724045b7e10"},"schema_version":"1.0"},"canonical_sha256":"c3464efc60b83ff5308466e03cc5ac547b41995dbdf14c473d2856826d4e9237","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:55:13.298721Z","signature_b64":"zBKITMptKC+7sL7mmMy5bCfIvZkTwV7TkfyEr5MprxF6XdS1yGfg/8WFiK2Q1OYm3yhcadLu/clKRQvDm4KVCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c3464efc60b83ff5308466e03cc5ac547b41995dbdf14c473d2856826d4e9237","last_reissued_at":"2026-07-05T07:55:13.298360Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:55:13.298360Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2309.10691","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:55:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"BTQM7zBENBmKkbDLgrUGyw1mYlpYxyU+EysvmQDqM/nPkA9U7AizfG2HURetrLZl/szo7W90RiiFHSdikVduBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T14:14:31.910361Z"},"content_sha256":"768918373693a0b742de3cb8c2b1f5ba8aeadf967a98965cd375f4eacffeff53","schema_version":"1.0","event_id":"sha256:768918373693a0b742de3cb8c2b1f5ba8aeadf967a98965cd375f4eacffeff53"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:YNDE57DAXA77KMEEM3QDZRNMKR","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"MINT: Evaluating LLMs in Multi-turn Interaction with Tools and Language Feedback","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Hao Peng, Heng Ji, Jiateng Liu, Lifan Yuan, Xingyao Wang, Yangyi Chen, Zihan Wang","submitted_at":"2023-09-19T15:25:42Z","abstract_excerpt":"To solve complex tasks, large language models (LLMs) often require multiple rounds of interactions with the user, sometimes assisted by external tools. However, current evaluation protocols often emphasize benchmark performance with single-turn exchanges, neglecting the nuanced interactions among the user, LLMs, and external tools, while also underestimating the importance of natural language feedback from users. These oversights contribute to discrepancies between research benchmark evaluations and real-world use cases. We introduce MINT, a benchmark that evaluates LLMs' ability to solve task"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.10691","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2309.10691/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:55:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"+Ivk0RAksO9HQO5OZDGR+G+Dh7mVGHqLnBtu/S10yznJLOpwo9wutcYBCEVH1hd4IVj4LmrL6vvTyAaluKByCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T14:14:31.910895Z"},"content_sha256":"bf536c6f33a8c392dfab98438008118bcf3ee866c135edeee010ff5be007537a","schema_version":"1.0","event_id":"sha256:bf536c6f33a8c392dfab98438008118bcf3ee866c135edeee010ff5be007537a"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/YNDE57DAXA77KMEEM3QDZRNMKR/bundle.json","state_url":"https://pith.science/pith/YNDE57DAXA77KMEEM3QDZRNMKR/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/YNDE57DAXA77KMEEM3QDZRNMKR/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T14:14:31Z","links":{"resolver":"https://pith.science/pith/YNDE57DAXA77KMEEM3QDZRNMKR","bundle":"https://pith.science/pith/YNDE57DAXA77KMEEM3QDZRNMKR/bundle.json","state":"https://pith.science/pith/YNDE57DAXA77KMEEM3QDZRNMKR/state.json","well_known_bundle":"https://pith.science/.well-known/pith/YNDE57DAXA77KMEEM3QDZRNMKR/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:YNDE57DAXA77KMEEM3QDZRNMKR","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"96d0a49155bf6b4b61d6848710bb0274eba18da77a2359e2d550b724045b7e10","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-19T15:25:42Z","title_canon_sha256":"cf3f89404843380027e7ad4dd376f4cb4143bac52ed473efa7112f58e0f62aa4"},"schema_version":"1.0","source":{"id":"2309.10691","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.10691","created_at":"2026-07-05T07:55:13Z"},{"alias_kind":"arxiv_version","alias_value":"2309.10691v3","created_at":"2026-07-05T07:55:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.10691","created_at":"2026-07-05T07:55:13Z"},{"alias_kind":"pith_short_12","alias_value":"YNDE57DAXA77","created_at":"2026-07-05T07:55:13Z"},{"alias_kind":"pith_short_16","alias_value":"YNDE57DAXA77KMEE","created_at":"2026-07-05T07:55:13Z"},{"alias_kind":"pith_short_8","alias_value":"YNDE57DA","created_at":"2026-07-05T07:55:13Z"}],"graph_snapshots":[{"event_id":"sha256:bf536c6f33a8c392dfab98438008118bcf3ee866c135edeee010ff5be007537a","target":"graph","created_at":"2026-07-05T07:55:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2309.10691/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"To solve complex tasks, large language models (LLMs) often require multiple rounds of interactions with the user, sometimes assisted by external tools. However, current evaluation protocols often emphasize benchmark performance with single-turn exchanges, neglecting the nuanced interactions among the user, LLMs, and external tools, while also underestimating the importance of natural language feedback from users. These oversights contribute to discrepancies between research benchmark evaluations and real-world use cases. We introduce MINT, a benchmark that evaluates LLMs' ability to solve task","authors_text":"Hao Peng, Heng Ji, Jiateng Liu, Lifan Yuan, Xingyao Wang, Yangyi Chen, Zihan Wang","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-19T15:25:42Z","title":"MINT: Evaluating LLMs in Multi-turn Interaction with Tools and Language Feedback"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.10691","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:768918373693a0b742de3cb8c2b1f5ba8aeadf967a98965cd375f4eacffeff53","target":"record","created_at":"2026-07-05T07:55:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"96d0a49155bf6b4b61d6848710bb0274eba18da77a2359e2d550b724045b7e10","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-19T15:25:42Z","title_canon_sha256":"cf3f89404843380027e7ad4dd376f4cb4143bac52ed473efa7112f58e0f62aa4"},"schema_version":"1.0","source":{"id":"2309.10691","kind":"arxiv","version":3}},"canonical_sha256":"c3464efc60b83ff5308466e03cc5ac547b41995dbdf14c473d2856826d4e9237","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c3464efc60b83ff5308466e03cc5ac547b41995dbdf14c473d2856826d4e9237","first_computed_at":"2026-07-05T07:55:13.298360Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:55:13.298360Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"zBKITMptKC+7sL7mmMy5bCfIvZkTwV7TkfyEr5MprxF6XdS1yGfg/8WFiK2Q1OYm3yhcadLu/clKRQvDm4KVCg==","signature_status":"signed_v1","signed_at":"2026-07-05T07:55:13.298721Z","signed_message":"canonical_sha256_bytes"},"source_id":"2309.10691","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:768918373693a0b742de3cb8c2b1f5ba8aeadf967a98965cd375f4eacffeff53","sha256:bf536c6f33a8c392dfab98438008118bcf3ee866c135edeee010ff5be007537a"],"state_sha256":"a5ba04a75835f51dcc68ae41c6c4523d422bd5425430692ad0f4dabf28ca77e5"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"7z/MKUHOCy6Ium9RboDvUsMCT3hDVjms2QIPvvNiEkaNwf7HtXVIzw9/gsxsHkl/VfVAMQmewldHSVi+a/ytDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T14:14:31.915907Z","bundle_sha256":"2f5c1ae2c19970a1fcc5e6b28775e4951fc4fcf7fa014ae43828fddc3b329581"}}