{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:VOJSRZ3AZ74HFZLF7ZAOMHP23U","short_pith_number":"pith:VOJSRZ3A","canonical_record":{"source":{"id":"2404.09785","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-15T13:40:08Z","cross_cats_sorted":[],"title_canon_sha256":"1201bc9caf27fcb8c8075f46dc3a305f64db4286150303fedf8b805ec258a210","abstract_canon_sha256":"6fa55b51fe4a1ae9e62b233968616ee776a9fea45ab138e5cc7fac6cad79a4db"},"schema_version":"1.0"},"canonical_sha256":"ab9328e760cff872e565fe40e61dfadd32a3e3ed53e600c41831e65013d884b0","source":{"kind":"arxiv","id":"2404.09785","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.09785","created_at":"2026-07-05T08:08:11Z"},{"alias_kind":"arxiv_version","alias_value":"2404.09785v1","created_at":"2026-07-05T08:08:11Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.09785","created_at":"2026-07-05T08:08:11Z"},{"alias_kind":"pith_short_12","alias_value":"VOJSRZ3AZ74H","created_at":"2026-07-05T08:08:11Z"},{"alias_kind":"pith_short_16","alias_value":"VOJSRZ3AZ74HFZLF","created_at":"2026-07-05T08:08:11Z"},{"alias_kind":"pith_short_8","alias_value":"VOJSRZ3A","created_at":"2026-07-05T08:08:11Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:VOJSRZ3AZ74HFZLF7ZAOMHP23U","target":"record","payload":{"canonical_record":{"source":{"id":"2404.09785","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-15T13:40:08Z","cross_cats_sorted":[],"title_canon_sha256":"1201bc9caf27fcb8c8075f46dc3a305f64db4286150303fedf8b805ec258a210","abstract_canon_sha256":"6fa55b51fe4a1ae9e62b233968616ee776a9fea45ab138e5cc7fac6cad79a4db"},"schema_version":"1.0"},"canonical_sha256":"ab9328e760cff872e565fe40e61dfadd32a3e3ed53e600c41831e65013d884b0","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:08:11.633196Z","signature_b64":"naIFIQF4fS3aP0KAtqCnRK5Nzu90tKPdndtIARimtfVlOwNgv1XCRYUmHbPSF/R4uztcWgcgj+jNb2bO7cogAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ab9328e760cff872e565fe40e61dfadd32a3e3ed53e600c41831e65013d884b0","last_reissued_at":"2026-07-05T08:08:11.632663Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:08:11.632663Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2404.09785","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:08:11Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"JKer7fk5jOxp21p8t0tiZp+1Yrq1DnfURwLowT270w6P+msaJANJ08rGpLd9flC9SECyKONjr9krYhvp2TdBDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T02:42:50.885148Z"},"content_sha256":"803a6d8d85741566ccd0f0862872fd81f8a173deadf2dc5f8eaf17bedbced015","schema_version":"1.0","event_id":"sha256:803a6d8d85741566ccd0f0862872fd81f8a173deadf2dc5f8eaf17bedbced015"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:VOJSRZ3AZ74HFZLF7ZAOMHP23U","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Benchmarking Llama2, Mistral, Gemma and GPT for Factuality, Toxicity, Bias and Propensity for Hallucinations","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"David Nadeau, Karen McNeil, Mike Kroutikov, Simon Baribeau","submitted_at":"2024-04-15T13:40:08Z","abstract_excerpt":"This paper introduces fourteen novel datasets for the evaluation of Large Language Models' safety in the context of enterprise tasks. A method was devised to evaluate a model's safety, as determined by its ability to follow instructions and output factual, unbiased, grounded, and appropriate content. In this research, we used OpenAI GPT as point of comparison since it excels at all levels of safety. On the open-source side, for smaller models, Meta Llama2 performs well at factuality and toxicity but has the highest propensity for hallucination. Mistral hallucinates the least but cannot handle "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.09785","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2404.09785/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:08:11Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"pEghi8bB53wZudP2tQIUsZUXIqFNxXNmaU09yEaGsglX3ZlIgpHg0Mw/yhhHHQXIVzlm+WoSQ63UDLF5sSCJDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T02:42:50.885576Z"},"content_sha256":"4fcf89ddf63d29be254a880d8c29c5aa42096fb081c5334ee59f4c3f46342e99","schema_version":"1.0","event_id":"sha256:4fcf89ddf63d29be254a880d8c29c5aa42096fb081c5334ee59f4c3f46342e99"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/VOJSRZ3AZ74HFZLF7ZAOMHP23U/bundle.json","state_url":"https://pith.science/pith/VOJSRZ3AZ74HFZLF7ZAOMHP23U/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/VOJSRZ3AZ74HFZLF7ZAOMHP23U/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-18T02:42:50Z","links":{"resolver":"https://pith.science/pith/VOJSRZ3AZ74HFZLF7ZAOMHP23U","bundle":"https://pith.science/pith/VOJSRZ3AZ74HFZLF7ZAOMHP23U/bundle.json","state":"https://pith.science/pith/VOJSRZ3AZ74HFZLF7ZAOMHP23U/state.json","well_known_bundle":"https://pith.science/.well-known/pith/VOJSRZ3AZ74HFZLF7ZAOMHP23U/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:VOJSRZ3AZ74HFZLF7ZAOMHP23U","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"6fa55b51fe4a1ae9e62b233968616ee776a9fea45ab138e5cc7fac6cad79a4db","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-15T13:40:08Z","title_canon_sha256":"1201bc9caf27fcb8c8075f46dc3a305f64db4286150303fedf8b805ec258a210"},"schema_version":"1.0","source":{"id":"2404.09785","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.09785","created_at":"2026-07-05T08:08:11Z"},{"alias_kind":"arxiv_version","alias_value":"2404.09785v1","created_at":"2026-07-05T08:08:11Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.09785","created_at":"2026-07-05T08:08:11Z"},{"alias_kind":"pith_short_12","alias_value":"VOJSRZ3AZ74H","created_at":"2026-07-05T08:08:11Z"},{"alias_kind":"pith_short_16","alias_value":"VOJSRZ3AZ74HFZLF","created_at":"2026-07-05T08:08:11Z"},{"alias_kind":"pith_short_8","alias_value":"VOJSRZ3A","created_at":"2026-07-05T08:08:11Z"}],"graph_snapshots":[{"event_id":"sha256:4fcf89ddf63d29be254a880d8c29c5aa42096fb081c5334ee59f4c3f46342e99","target":"graph","created_at":"2026-07-05T08:08:11Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2404.09785/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"This paper introduces fourteen novel datasets for the evaluation of Large Language Models' safety in the context of enterprise tasks. A method was devised to evaluate a model's safety, as determined by its ability to follow instructions and output factual, unbiased, grounded, and appropriate content. In this research, we used OpenAI GPT as point of comparison since it excels at all levels of safety. On the open-source side, for smaller models, Meta Llama2 performs well at factuality and toxicity but has the highest propensity for hallucination. Mistral hallucinates the least but cannot handle ","authors_text":"David Nadeau, Karen McNeil, Mike Kroutikov, Simon Baribeau","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-15T13:40:08Z","title":"Benchmarking Llama2, Mistral, Gemma and GPT for Factuality, Toxicity, Bias and Propensity for Hallucinations"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.09785","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:803a6d8d85741566ccd0f0862872fd81f8a173deadf2dc5f8eaf17bedbced015","target":"record","created_at":"2026-07-05T08:08:11Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"6fa55b51fe4a1ae9e62b233968616ee776a9fea45ab138e5cc7fac6cad79a4db","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-15T13:40:08Z","title_canon_sha256":"1201bc9caf27fcb8c8075f46dc3a305f64db4286150303fedf8b805ec258a210"},"schema_version":"1.0","source":{"id":"2404.09785","kind":"arxiv","version":1}},"canonical_sha256":"ab9328e760cff872e565fe40e61dfadd32a3e3ed53e600c41831e65013d884b0","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"ab9328e760cff872e565fe40e61dfadd32a3e3ed53e600c41831e65013d884b0","first_computed_at":"2026-07-05T08:08:11.632663Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:08:11.632663Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"naIFIQF4fS3aP0KAtqCnRK5Nzu90tKPdndtIARimtfVlOwNgv1XCRYUmHbPSF/R4uztcWgcgj+jNb2bO7cogAA==","signature_status":"signed_v1","signed_at":"2026-07-05T08:08:11.633196Z","signed_message":"canonical_sha256_bytes"},"source_id":"2404.09785","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:803a6d8d85741566ccd0f0862872fd81f8a173deadf2dc5f8eaf17bedbced015","sha256:4fcf89ddf63d29be254a880d8c29c5aa42096fb081c5334ee59f4c3f46342e99"],"state_sha256":"8cadd29a217dc34fc9b555c8a70f23d497158f506778e72d941f38a882d63fc9"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"JTtYebJR+Q9xYw1QZbaqTD7AX+fVnNhF/BVXZ/Pm6u0H39/t5xINZf1dDYIddDyXytbs9giQzqN7jOW8K1OyCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-18T02:42:50.887947Z","bundle_sha256":"38dc563b6c86c295d911f5c14607ea7037d4459b5396abb4343a25c0e9806473"}}