{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:VWELPAVGE5FMPSFUQ7QAIY3CS3","short_pith_number":"pith:VWELPAVG","canonical_record":{"source":{"id":"2405.14766","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-23T16:33:18Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"85904329a18e1b482750b08f7f2cee40f2881869c610531d3c39bb4772d4ea27","abstract_canon_sha256":"7bccaecb941d4031f49f7690179ff8f19a4822c6fae289626a43b217791d8236"},"schema_version":"1.0"},"canonical_sha256":"ad88b782a6274ac7c8b487e004636296ffb8df70ed491d70d3eebef330f5363a","source":{"kind":"arxiv","id":"2405.14766","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.14766","created_at":"2026-07-05T10:16:58Z"},{"alias_kind":"arxiv_version","alias_value":"2405.14766v2","created_at":"2026-07-05T10:16:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.14766","created_at":"2026-07-05T10:16:58Z"},{"alias_kind":"pith_short_12","alias_value":"VWELPAVGE5FM","created_at":"2026-07-05T10:16:58Z"},{"alias_kind":"pith_short_16","alias_value":"VWELPAVGE5FMPSFU","created_at":"2026-07-05T10:16:58Z"},{"alias_kind":"pith_short_8","alias_value":"VWELPAVG","created_at":"2026-07-05T10:16:58Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:VWELPAVGE5FMPSFUQ7QAIY3CS3","target":"record","payload":{"canonical_record":{"source":{"id":"2405.14766","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-23T16:33:18Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"85904329a18e1b482750b08f7f2cee40f2881869c610531d3c39bb4772d4ea27","abstract_canon_sha256":"7bccaecb941d4031f49f7690179ff8f19a4822c6fae289626a43b217791d8236"},"schema_version":"1.0"},"canonical_sha256":"ad88b782a6274ac7c8b487e004636296ffb8df70ed491d70d3eebef330f5363a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:16:58.445223Z","signature_b64":"MF0C1WXw4SasARKcmx6+E0IOrFEFEEnYJBZtmooSzgjsqd7lccC4995rcy/kbPZNhlNWI6nFjTCWsN7GTF0jAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ad88b782a6274ac7c8b487e004636296ffb8df70ed491d70d3eebef330f5363a","last_reissued_at":"2026-07-05T10:16:58.444660Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:16:58.444660Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.14766","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:16:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"PcNMJbZ2Pc3aroNrpPTYCVG5BlX+Gt4u94B5Ytm5xE5ATDGFC0wpVUWOCN/2b07jaevV1JP2kkLWQYUNvBi0DA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-14T20:15:21.105589Z"},"content_sha256":"fa463256bf870eb5674eaef78474c3a96176c9f6443d95fd5da12949b6bb671d","schema_version":"1.0","event_id":"sha256:fa463256bf870eb5674eaef78474c3a96176c9f6443d95fd5da12949b6bb671d"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:VWELPAVGE5FMPSFUQ7QAIY3CS3","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Evaluating Large Language Models for Public Health Classification and Extraction Tasks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Amy Douglas, Fan Grayson, Galena Kuyumdzhieva, Hamish Mohammed, Harry Long, Holly Fountain, Jo Hardstaff, Joshua Harris, Kathryn Hopkins, Leo Loman, Lesley Larkin, Loes WalsGriffith, Luke Hounsome, Michael Borowitz, Samuel Collins, Stelios Georgiou, Steven Riley, Thomas Finnie, Timothy Laurence, Toby Nonnenmacher, Y-Ling Chi","submitted_at":"2024-05-23T16:33:18Z","abstract_excerpt":"Advances in Large Language Models (LLMs) have led to significant interest in their potential to support human experts across a range of domains, including public health. In this work we present automated evaluations of LLMs for public health tasks involving the classification and extraction of free text. We combine six externally annotated datasets with seven new internally annotated datasets to evaluate LLMs for processing text related to: health burden, epidemiological risk factors, and public health interventions. We evaluate eleven open-weight LLMs (7-123 billion parameters) across all tas"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.14766","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.14766/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:16:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"zgYL1K/1bhCS+Y/z4O3bVhTw7ZmtrIAZEo+DQImjny3tG+ygI3zUo4/u1HH6VdSTRTXZyAmPFv33s6x5jWF1Dg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-14T20:15:21.106110Z"},"content_sha256":"945fcf6e89e0b5b8510eb406e9910c03c021438a1cf08f4641c7fd8ef59f1d73","schema_version":"1.0","event_id":"sha256:945fcf6e89e0b5b8510eb406e9910c03c021438a1cf08f4641c7fd8ef59f1d73"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/VWELPAVGE5FMPSFUQ7QAIY3CS3/bundle.json","state_url":"https://pith.science/pith/VWELPAVGE5FMPSFUQ7QAIY3CS3/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/VWELPAVGE5FMPSFUQ7QAIY3CS3/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-14T20:15:21Z","links":{"resolver":"https://pith.science/pith/VWELPAVGE5FMPSFUQ7QAIY3CS3","bundle":"https://pith.science/pith/VWELPAVGE5FMPSFUQ7QAIY3CS3/bundle.json","state":"https://pith.science/pith/VWELPAVGE5FMPSFUQ7QAIY3CS3/state.json","well_known_bundle":"https://pith.science/.well-known/pith/VWELPAVGE5FMPSFUQ7QAIY3CS3/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:VWELPAVGE5FMPSFUQ7QAIY3CS3","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7bccaecb941d4031f49f7690179ff8f19a4822c6fae289626a43b217791d8236","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-23T16:33:18Z","title_canon_sha256":"85904329a18e1b482750b08f7f2cee40f2881869c610531d3c39bb4772d4ea27"},"schema_version":"1.0","source":{"id":"2405.14766","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.14766","created_at":"2026-07-05T10:16:58Z"},{"alias_kind":"arxiv_version","alias_value":"2405.14766v2","created_at":"2026-07-05T10:16:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.14766","created_at":"2026-07-05T10:16:58Z"},{"alias_kind":"pith_short_12","alias_value":"VWELPAVGE5FM","created_at":"2026-07-05T10:16:58Z"},{"alias_kind":"pith_short_16","alias_value":"VWELPAVGE5FMPSFU","created_at":"2026-07-05T10:16:58Z"},{"alias_kind":"pith_short_8","alias_value":"VWELPAVG","created_at":"2026-07-05T10:16:58Z"}],"graph_snapshots":[{"event_id":"sha256:945fcf6e89e0b5b8510eb406e9910c03c021438a1cf08f4641c7fd8ef59f1d73","target":"graph","created_at":"2026-07-05T10:16:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.14766/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Advances in Large Language Models (LLMs) have led to significant interest in their potential to support human experts across a range of domains, including public health. In this work we present automated evaluations of LLMs for public health tasks involving the classification and extraction of free text. We combine six externally annotated datasets with seven new internally annotated datasets to evaluate LLMs for processing text related to: health burden, epidemiological risk factors, and public health interventions. We evaluate eleven open-weight LLMs (7-123 billion parameters) across all tas","authors_text":"Amy Douglas, Fan Grayson, Galena Kuyumdzhieva, Hamish Mohammed, Harry Long, Holly Fountain, Jo Hardstaff, Joshua Harris, Kathryn Hopkins, Leo Loman, Lesley Larkin, Loes WalsGriffith, Luke Hounsome, Michael Borowitz, Samuel Collins, Stelios Georgiou, Steven Riley, Thomas Finnie, Timothy Laurence, Toby Nonnenmacher, Y-Ling Chi","cross_cats":["cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-23T16:33:18Z","title":"Evaluating Large Language Models for Public Health Classification and Extraction Tasks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.14766","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:fa463256bf870eb5674eaef78474c3a96176c9f6443d95fd5da12949b6bb671d","target":"record","created_at":"2026-07-05T10:16:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7bccaecb941d4031f49f7690179ff8f19a4822c6fae289626a43b217791d8236","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-23T16:33:18Z","title_canon_sha256":"85904329a18e1b482750b08f7f2cee40f2881869c610531d3c39bb4772d4ea27"},"schema_version":"1.0","source":{"id":"2405.14766","kind":"arxiv","version":2}},"canonical_sha256":"ad88b782a6274ac7c8b487e004636296ffb8df70ed491d70d3eebef330f5363a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"ad88b782a6274ac7c8b487e004636296ffb8df70ed491d70d3eebef330f5363a","first_computed_at":"2026-07-05T10:16:58.444660Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:16:58.444660Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"MF0C1WXw4SasARKcmx6+E0IOrFEFEEnYJBZtmooSzgjsqd7lccC4995rcy/kbPZNhlNWI6nFjTCWsN7GTF0jAA==","signature_status":"signed_v1","signed_at":"2026-07-05T10:16:58.445223Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.14766","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:fa463256bf870eb5674eaef78474c3a96176c9f6443d95fd5da12949b6bb671d","sha256:945fcf6e89e0b5b8510eb406e9910c03c021438a1cf08f4641c7fd8ef59f1d73"],"state_sha256":"6a51a97ac88bc60456e2e418d9475813a59b7603abe5391690d6e4085e4fae18"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"kDr19ndL/JIl9Fk4TevVrONLSCivnfPDc280L4B3uoTlX9WVPbuqd59CpwizwJl2/U1HYitlGmtdXWFs9LkcBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-14T20:15:21.110002Z","bundle_sha256":"8d96abbd9880af81ca9baf41bbb37797d97646c315ca565b64af5555d71a0c1a"}}