{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:HNHQHQWW6AUIF4QVQHBKW3NV4I","short_pith_number":"pith:HNHQHQWW","canonical_record":{"source":{"id":"2406.04520","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-06T21:27:35Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"801173d26fab18080eb47b27d08fd9becf18ee0a74b5ef7083319b18b60efbd6","abstract_canon_sha256":"537994289fa500d3af66b7d3f5ff6e08149a2f188843b6b5181838af41e631fc"},"schema_version":"1.0"},"canonical_sha256":"3b4f03c2d6f02882f21581c2ab6db5e200ebd3f3c606bada3f0fe47cbc941c99","source":{"kind":"arxiv","id":"2406.04520","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.04520","created_at":"2026-07-05T08:28:44Z"},{"alias_kind":"arxiv_version","alias_value":"2406.04520v1","created_at":"2026-07-05T08:28:44Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.04520","created_at":"2026-07-05T08:28:44Z"},{"alias_kind":"pith_short_12","alias_value":"HNHQHQWW6AUI","created_at":"2026-07-05T08:28:44Z"},{"alias_kind":"pith_short_16","alias_value":"HNHQHQWW6AUIF4QV","created_at":"2026-07-05T08:28:44Z"},{"alias_kind":"pith_short_8","alias_value":"HNHQHQWW","created_at":"2026-07-05T08:28:44Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:HNHQHQWW6AUIF4QVQHBKW3NV4I","target":"record","payload":{"canonical_record":{"source":{"id":"2406.04520","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-06T21:27:35Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"801173d26fab18080eb47b27d08fd9becf18ee0a74b5ef7083319b18b60efbd6","abstract_canon_sha256":"537994289fa500d3af66b7d3f5ff6e08149a2f188843b6b5181838af41e631fc"},"schema_version":"1.0"},"canonical_sha256":"3b4f03c2d6f02882f21581c2ab6db5e200ebd3f3c606bada3f0fe47cbc941c99","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:28:44.929066Z","signature_b64":"qW4QlRO4Wqa0gWjlSM/I+tqqOsZ8HETtrcenj5QbltWgFjZRrRzjI0nl5q9IjxFuK0TcVYyJnXM/T54Q/RtIBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3b4f03c2d6f02882f21581c2ab6db5e200ebd3f3c606bada3f0fe47cbc941c99","last_reissued_at":"2026-07-05T08:28:44.928557Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:28:44.928557Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.04520","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:28:44Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"OhPxKMuc8GfhJz4x9K1MZfEJUFtQ5bTM8GZua9vJHdBTbAiTpf8hKaz8tKZ+JUlf2k+mF9zcT6wbcwLKHE8qCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-22T06:31:44.102617Z"},"content_sha256":"47f7710196493e1e29981b14eeb4a4093cb505af9e86e2bd13c815d231bb0fa5","schema_version":"1.0","event_id":"sha256:47f7710196493e1e29981b14eeb4a4093cb505af9e86e2bd13c815d231bb0fa5"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:HNHQHQWW6AUIF4QVQHBKW3NV4I","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"NATURAL PLAN: Benchmarking LLMs on Natural Language Planning","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Azade Nova, Denny Zhou, Ed H. Chi, Heng-Tze Cheng, Huaixiu Steven Zheng, Hugh Zhang, Le Hou, Minmin Chen, Quoc V. Le, Swaroop Mishra, Xinyun Chen","submitted_at":"2024-06-06T21:27:35Z","abstract_excerpt":"We introduce NATURAL PLAN, a realistic planning benchmark in natural language containing 3 key tasks: Trip Planning, Meeting Planning, and Calendar Scheduling. We focus our evaluation on the planning capabilities of LLMs with full information on the task, by providing outputs from tools such as Google Flights, Google Maps, and Google Calendar as contexts to the models. This eliminates the need for a tool-use environment for evaluating LLMs on Planning. We observe that NATURAL PLAN is a challenging benchmark for state of the art models. For example, in Trip Planning, GPT-4 and Gemini 1.5 Pro co"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.04520","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.04520/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:28:44Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"YNxkXQbcQ0/z38HIUTt4c/TvnsEtMkoO9Phn2oVAeVz0j3g0B9MGFmNWM63BzAXeNkPklCOQOdCHA3SwBusKBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-22T06:31:44.103104Z"},"content_sha256":"fb39c52bb79ecb7dff23586ba956356ffc38a9abd96b8e0978bf380e98071fd0","schema_version":"1.0","event_id":"sha256:fb39c52bb79ecb7dff23586ba956356ffc38a9abd96b8e0978bf380e98071fd0"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/HNHQHQWW6AUIF4QVQHBKW3NV4I/bundle.json","state_url":"https://pith.science/pith/HNHQHQWW6AUIF4QVQHBKW3NV4I/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/HNHQHQWW6AUIF4QVQHBKW3NV4I/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-22T06:31:44Z","links":{"resolver":"https://pith.science/pith/HNHQHQWW6AUIF4QVQHBKW3NV4I","bundle":"https://pith.science/pith/HNHQHQWW6AUIF4QVQHBKW3NV4I/bundle.json","state":"https://pith.science/pith/HNHQHQWW6AUIF4QVQHBKW3NV4I/state.json","well_known_bundle":"https://pith.science/.well-known/pith/HNHQHQWW6AUIF4QVQHBKW3NV4I/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:HNHQHQWW6AUIF4QVQHBKW3NV4I","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"537994289fa500d3af66b7d3f5ff6e08149a2f188843b6b5181838af41e631fc","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-06T21:27:35Z","title_canon_sha256":"801173d26fab18080eb47b27d08fd9becf18ee0a74b5ef7083319b18b60efbd6"},"schema_version":"1.0","source":{"id":"2406.04520","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.04520","created_at":"2026-07-05T08:28:44Z"},{"alias_kind":"arxiv_version","alias_value":"2406.04520v1","created_at":"2026-07-05T08:28:44Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.04520","created_at":"2026-07-05T08:28:44Z"},{"alias_kind":"pith_short_12","alias_value":"HNHQHQWW6AUI","created_at":"2026-07-05T08:28:44Z"},{"alias_kind":"pith_short_16","alias_value":"HNHQHQWW6AUIF4QV","created_at":"2026-07-05T08:28:44Z"},{"alias_kind":"pith_short_8","alias_value":"HNHQHQWW","created_at":"2026-07-05T08:28:44Z"}],"graph_snapshots":[{"event_id":"sha256:fb39c52bb79ecb7dff23586ba956356ffc38a9abd96b8e0978bf380e98071fd0","target":"graph","created_at":"2026-07-05T08:28:44Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.04520/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We introduce NATURAL PLAN, a realistic planning benchmark in natural language containing 3 key tasks: Trip Planning, Meeting Planning, and Calendar Scheduling. We focus our evaluation on the planning capabilities of LLMs with full information on the task, by providing outputs from tools such as Google Flights, Google Maps, and Google Calendar as contexts to the models. This eliminates the need for a tool-use environment for evaluating LLMs on Planning. We observe that NATURAL PLAN is a challenging benchmark for state of the art models. For example, in Trip Planning, GPT-4 and Gemini 1.5 Pro co","authors_text":"Azade Nova, Denny Zhou, Ed H. Chi, Heng-Tze Cheng, Huaixiu Steven Zheng, Hugh Zhang, Le Hou, Minmin Chen, Quoc V. Le, Swaroop Mishra, Xinyun Chen","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-06T21:27:35Z","title":"NATURAL PLAN: Benchmarking LLMs on Natural Language Planning"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.04520","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:47f7710196493e1e29981b14eeb4a4093cb505af9e86e2bd13c815d231bb0fa5","target":"record","created_at":"2026-07-05T08:28:44Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"537994289fa500d3af66b7d3f5ff6e08149a2f188843b6b5181838af41e631fc","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-06T21:27:35Z","title_canon_sha256":"801173d26fab18080eb47b27d08fd9becf18ee0a74b5ef7083319b18b60efbd6"},"schema_version":"1.0","source":{"id":"2406.04520","kind":"arxiv","version":1}},"canonical_sha256":"3b4f03c2d6f02882f21581c2ab6db5e200ebd3f3c606bada3f0fe47cbc941c99","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3b4f03c2d6f02882f21581c2ab6db5e200ebd3f3c606bada3f0fe47cbc941c99","first_computed_at":"2026-07-05T08:28:44.928557Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:28:44.928557Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"qW4QlRO4Wqa0gWjlSM/I+tqqOsZ8HETtrcenj5QbltWgFjZRrRzjI0nl5q9IjxFuK0TcVYyJnXM/T54Q/RtIBw==","signature_status":"signed_v1","signed_at":"2026-07-05T08:28:44.929066Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.04520","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:47f7710196493e1e29981b14eeb4a4093cb505af9e86e2bd13c815d231bb0fa5","sha256:fb39c52bb79ecb7dff23586ba956356ffc38a9abd96b8e0978bf380e98071fd0"],"state_sha256":"aa7d453495fd4f2fc6cc2bf07896734287daec890294bbe3a02788164f4c6e4e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"B+qEIuB17DboKnkWlolV+8UA/STFAFKnQ0ibq6jJEMEpDRMTA6GBcqf+VtLph6jeLZIe1/MgVKCAJ1F4qT1mAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-22T06:31:44.107933Z","bundle_sha256":"4d8e74f419d040912e3cfc286bbca39c86dc06d35526bdaa30f4070981b5b068"}}