{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:YLCTQ7HYILKMO2DZUTU7TZHBXY","short_pith_number":"pith:YLCTQ7HY","canonical_record":{"source":{"id":"2310.10632","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-16T17:54:20Z","cross_cats_sorted":["cs.AI","cs.RO"],"title_canon_sha256":"7c3d4a92f0485ac515f47fb9571e39a9f9b269ff2254ab892942c201358b1673","abstract_canon_sha256":"e936a33b5ee7d1d87d28fa7dd9578f481d8d7c2f03a1c60d93ab014f096eadf5"},"schema_version":"1.0"},"canonical_sha256":"c2c5387cf842d4c76879a4e9f9e4e1be38e10512d9ae424e01cef834d0e9a472","source":{"kind":"arxiv","id":"2310.10632","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.10632","created_at":"2026-07-05T07:01:22Z"},{"alias_kind":"arxiv_version","alias_value":"2310.10632v1","created_at":"2026-07-05T07:01:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.10632","created_at":"2026-07-05T07:01:22Z"},{"alias_kind":"pith_short_12","alias_value":"YLCTQ7HYILKM","created_at":"2026-07-05T07:01:22Z"},{"alias_kind":"pith_short_16","alias_value":"YLCTQ7HYILKMO2DZ","created_at":"2026-07-05T07:01:22Z"},{"alias_kind":"pith_short_8","alias_value":"YLCTQ7HY","created_at":"2026-07-05T07:01:22Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:YLCTQ7HYILKMO2DZUTU7TZHBXY","target":"record","payload":{"canonical_record":{"source":{"id":"2310.10632","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-16T17:54:20Z","cross_cats_sorted":["cs.AI","cs.RO"],"title_canon_sha256":"7c3d4a92f0485ac515f47fb9571e39a9f9b269ff2254ab892942c201358b1673","abstract_canon_sha256":"e936a33b5ee7d1d87d28fa7dd9578f481d8d7c2f03a1c60d93ab014f096eadf5"},"schema_version":"1.0"},"canonical_sha256":"c2c5387cf842d4c76879a4e9f9e4e1be38e10512d9ae424e01cef834d0e9a472","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:01:22.661339Z","signature_b64":"WG4UR/nf/ylxGLhvqq95+QBw3OMUL9PLYH84i6EaGtWB1nVR8kHxa9SKfEASmhEH7dsHqaIQf1QxqXMhZvdgDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c2c5387cf842d4c76879a4e9f9e4e1be38e10512d9ae424e01cef834d0e9a472","last_reissued_at":"2026-07-05T07:01:22.660736Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:01:22.660736Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2310.10632","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:01:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"igquDAPR52osPRiSD81OMxkCjWN66r0GyiPSiGg0F/QG7uc4JsM/j2Z86yvsVYESFluSXnYeXLz/AOx9gUdMAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T16:17:33.916205Z"},"content_sha256":"25d12062c4f649cbfd1f90ef695988234ae4b7ae5a1d9f59ea91a489ed28e1c1","schema_version":"1.0","event_id":"sha256:25d12062c4f649cbfd1f90ef695988234ae4b7ae5a1d9f59ea91a489ed28e1c1"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:YLCTQ7HYILKMO2DZUTU7TZHBXY","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"BioPlanner: Automatic Evaluation of LLMs on Protocol Planning in Biology","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.RO"],"primary_cat":"cs.CL","authors_text":"Aleksandar Shtedritski, Ali Essa Ghareeb, John Ginger, Justin Booth, Odhran O'Donoghue, Ralph Abboud, Samuel G Rodriques","submitted_at":"2023-10-16T17:54:20Z","abstract_excerpt":"The ability to automatically generate accurate protocols for scientific experiments would represent a major step towards the automation of science. Large Language Models (LLMs) have impressive capabilities on a wide range of tasks, such as question answering and the generation of coherent text and code. However, LLMs can struggle with multi-step problems and long-term planning, which are crucial for designing scientific experiments. Moreover, evaluation of the accuracy of scientific protocols is challenging, because experiments can be described correctly in many different ways, require expert "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.10632","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2310.10632/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:01:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"9/rslrvhF4bgXAIjcUm/dod06ixorRuSO7YM96iN1fyQpn1Pu/FpdrZ/kfLa+N+on1yCS0L+shK+P4hQite9Bg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T16:17:33.916709Z"},"content_sha256":"67c57238f9e0b709f4b3b077bf1ff47803a8dc0ed56cfb6d4b5902c0a0d51aa0","schema_version":"1.0","event_id":"sha256:67c57238f9e0b709f4b3b077bf1ff47803a8dc0ed56cfb6d4b5902c0a0d51aa0"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/YLCTQ7HYILKMO2DZUTU7TZHBXY/bundle.json","state_url":"https://pith.science/pith/YLCTQ7HYILKMO2DZUTU7TZHBXY/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/YLCTQ7HYILKMO2DZUTU7TZHBXY/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T16:17:33Z","links":{"resolver":"https://pith.science/pith/YLCTQ7HYILKMO2DZUTU7TZHBXY","bundle":"https://pith.science/pith/YLCTQ7HYILKMO2DZUTU7TZHBXY/bundle.json","state":"https://pith.science/pith/YLCTQ7HYILKMO2DZUTU7TZHBXY/state.json","well_known_bundle":"https://pith.science/.well-known/pith/YLCTQ7HYILKMO2DZUTU7TZHBXY/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:YLCTQ7HYILKMO2DZUTU7TZHBXY","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"e936a33b5ee7d1d87d28fa7dd9578f481d8d7c2f03a1c60d93ab014f096eadf5","cross_cats_sorted":["cs.AI","cs.RO"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-16T17:54:20Z","title_canon_sha256":"7c3d4a92f0485ac515f47fb9571e39a9f9b269ff2254ab892942c201358b1673"},"schema_version":"1.0","source":{"id":"2310.10632","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.10632","created_at":"2026-07-05T07:01:22Z"},{"alias_kind":"arxiv_version","alias_value":"2310.10632v1","created_at":"2026-07-05T07:01:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.10632","created_at":"2026-07-05T07:01:22Z"},{"alias_kind":"pith_short_12","alias_value":"YLCTQ7HYILKM","created_at":"2026-07-05T07:01:22Z"},{"alias_kind":"pith_short_16","alias_value":"YLCTQ7HYILKMO2DZ","created_at":"2026-07-05T07:01:22Z"},{"alias_kind":"pith_short_8","alias_value":"YLCTQ7HY","created_at":"2026-07-05T07:01:22Z"}],"graph_snapshots":[{"event_id":"sha256:67c57238f9e0b709f4b3b077bf1ff47803a8dc0ed56cfb6d4b5902c0a0d51aa0","target":"graph","created_at":"2026-07-05T07:01:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2310.10632/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The ability to automatically generate accurate protocols for scientific experiments would represent a major step towards the automation of science. Large Language Models (LLMs) have impressive capabilities on a wide range of tasks, such as question answering and the generation of coherent text and code. However, LLMs can struggle with multi-step problems and long-term planning, which are crucial for designing scientific experiments. Moreover, evaluation of the accuracy of scientific protocols is challenging, because experiments can be described correctly in many different ways, require expert ","authors_text":"Aleksandar Shtedritski, Ali Essa Ghareeb, John Ginger, Justin Booth, Odhran O'Donoghue, Ralph Abboud, Samuel G Rodriques","cross_cats":["cs.AI","cs.RO"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-16T17:54:20Z","title":"BioPlanner: Automatic Evaluation of LLMs on Protocol Planning in Biology"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.10632","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:25d12062c4f649cbfd1f90ef695988234ae4b7ae5a1d9f59ea91a489ed28e1c1","target":"record","created_at":"2026-07-05T07:01:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"e936a33b5ee7d1d87d28fa7dd9578f481d8d7c2f03a1c60d93ab014f096eadf5","cross_cats_sorted":["cs.AI","cs.RO"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-16T17:54:20Z","title_canon_sha256":"7c3d4a92f0485ac515f47fb9571e39a9f9b269ff2254ab892942c201358b1673"},"schema_version":"1.0","source":{"id":"2310.10632","kind":"arxiv","version":1}},"canonical_sha256":"c2c5387cf842d4c76879a4e9f9e4e1be38e10512d9ae424e01cef834d0e9a472","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c2c5387cf842d4c76879a4e9f9e4e1be38e10512d9ae424e01cef834d0e9a472","first_computed_at":"2026-07-05T07:01:22.660736Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:01:22.660736Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"WG4UR/nf/ylxGLhvqq95+QBw3OMUL9PLYH84i6EaGtWB1nVR8kHxa9SKfEASmhEH7dsHqaIQf1QxqXMhZvdgDg==","signature_status":"signed_v1","signed_at":"2026-07-05T07:01:22.661339Z","signed_message":"canonical_sha256_bytes"},"source_id":"2310.10632","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:25d12062c4f649cbfd1f90ef695988234ae4b7ae5a1d9f59ea91a489ed28e1c1","sha256:67c57238f9e0b709f4b3b077bf1ff47803a8dc0ed56cfb6d4b5902c0a0d51aa0"],"state_sha256":"fef3d46f129b95cf633633996a3cf105bbeae63fc1c42c740c67bb4141e22d6c"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jF0xafYbGf+uqHE0CtqkeU2M/gjyNS1+E0vh3fwf27GbacuuvgT5sNEAh/IfURe0JjlsvKL5kOsi4FLrlunwCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T16:17:33.920967Z","bundle_sha256":"31114f577de532987e5aa4a233ba857bb350848e35fbaf49d400dac264e10f36"}}