{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:6OW6THKG5VIE63IHTU46KKY6GA","short_pith_number":"pith:6OW6THKG","canonical_record":{"source":{"id":"2406.00343","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-01T07:36:59Z","cross_cats_sorted":[],"title_canon_sha256":"d35d622bc47f7af514a31a50a747ad4e623b24e04ecd6f3b4512f2e9253a923e","abstract_canon_sha256":"19c122f324c440c25fcb35f37d6abd4d4a6966e854b0ad49d06de69595384851"},"schema_version":"1.0"},"canonical_sha256":"f3ade99d46ed504f6d079d39e52b1e3031a77699e2f4c91cc3940ee37370cba5","source":{"kind":"arxiv","id":"2406.00343","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.00343","created_at":"2026-07-05T08:31:16Z"},{"alias_kind":"arxiv_version","alias_value":"2406.00343v2","created_at":"2026-07-05T08:31:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.00343","created_at":"2026-07-05T08:31:16Z"},{"alias_kind":"pith_short_12","alias_value":"6OW6THKG5VIE","created_at":"2026-07-05T08:31:16Z"},{"alias_kind":"pith_short_16","alias_value":"6OW6THKG5VIE63IH","created_at":"2026-07-05T08:31:16Z"},{"alias_kind":"pith_short_8","alias_value":"6OW6THKG","created_at":"2026-07-05T08:31:16Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:6OW6THKG5VIE63IHTU46KKY6GA","target":"record","payload":{"canonical_record":{"source":{"id":"2406.00343","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-01T07:36:59Z","cross_cats_sorted":[],"title_canon_sha256":"d35d622bc47f7af514a31a50a747ad4e623b24e04ecd6f3b4512f2e9253a923e","abstract_canon_sha256":"19c122f324c440c25fcb35f37d6abd4d4a6966e854b0ad49d06de69595384851"},"schema_version":"1.0"},"canonical_sha256":"f3ade99d46ed504f6d079d39e52b1e3031a77699e2f4c91cc3940ee37370cba5","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:31:16.012990Z","signature_b64":"fOWU4/Ze6k+/EPw+Ch1g/852oJcTjhLegSddGEj4kZbPtYASP0IhX7PDAhm9CshxgPQQFYC5XFkumnqGk3EUAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f3ade99d46ed504f6d079d39e52b1e3031a77699e2f4c91cc3940ee37370cba5","last_reissued_at":"2026-07-05T08:31:16.012586Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:31:16.012586Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.00343","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:31:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"2d7/ehPh2FbFKgwafacJ2HlE5R1Kj21ZiWumbTGlnutHeZhnizT2v/y35rwTMHtdSKBtEOB381hSjyRUXOfKDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T05:22:23.017511Z"},"content_sha256":"84773ca53dab07cfda2326c21d26cc6ffceff08ca4829b5b0365b4a533ff7ee4","schema_version":"1.0","event_id":"sha256:84773ca53dab07cfda2326c21d26cc6ffceff08ca4829b5b0365b4a533ff7ee4"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:6OW6THKG5VIE63IHTU46KKY6GA","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Beyond Metrics: Evaluating LLMs' Effectiveness in Culturally Nuanced, Low-Resource Real-World Scenarios","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Jacki O'Neill, Jindong Wang, Kalika Bali, Keshet Ronen, Millicent Ochieng, Sunayana Sitaram, Varun Gumma, Vishrav Chaudhary","submitted_at":"2024-06-01T07:36:59Z","abstract_excerpt":"The deployment of Large Language Models (LLMs) in real-world applications presents both opportunities and challenges, particularly in multilingual and code-mixed communication settings. This research evaluates the performance of seven leading LLMs in sentiment analysis on a dataset derived from multilingual and code-mixed WhatsApp chats, including Swahili, English and Sheng. Our evaluation includes both quantitative analysis using metrics like F1 score and qualitative assessment of LLMs' explanations for their predictions. We find that, while Mistral-7b and Mixtral-8x7b achieved high F1 scores"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.00343","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.00343/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:31:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"xbizyIxmo/ppBRls5XgJttfjOJ8qOfAaHX3pCEQ7MGYsWuIz0KM0/9/hlTYHaaAwTPdsCA9HzHWreYiXDLQvDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T05:22:23.018010Z"},"content_sha256":"801b507d57abff79a32a88a371407a7c01e13a27eaff7d8b1b7cd5bd244257b4","schema_version":"1.0","event_id":"sha256:801b507d57abff79a32a88a371407a7c01e13a27eaff7d8b1b7cd5bd244257b4"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/6OW6THKG5VIE63IHTU46KKY6GA/bundle.json","state_url":"https://pith.science/pith/6OW6THKG5VIE63IHTU46KKY6GA/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/6OW6THKG5VIE63IHTU46KKY6GA/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T05:22:23Z","links":{"resolver":"https://pith.science/pith/6OW6THKG5VIE63IHTU46KKY6GA","bundle":"https://pith.science/pith/6OW6THKG5VIE63IHTU46KKY6GA/bundle.json","state":"https://pith.science/pith/6OW6THKG5VIE63IHTU46KKY6GA/state.json","well_known_bundle":"https://pith.science/.well-known/pith/6OW6THKG5VIE63IHTU46KKY6GA/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:6OW6THKG5VIE63IHTU46KKY6GA","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"19c122f324c440c25fcb35f37d6abd4d4a6966e854b0ad49d06de69595384851","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-01T07:36:59Z","title_canon_sha256":"d35d622bc47f7af514a31a50a747ad4e623b24e04ecd6f3b4512f2e9253a923e"},"schema_version":"1.0","source":{"id":"2406.00343","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.00343","created_at":"2026-07-05T08:31:16Z"},{"alias_kind":"arxiv_version","alias_value":"2406.00343v2","created_at":"2026-07-05T08:31:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.00343","created_at":"2026-07-05T08:31:16Z"},{"alias_kind":"pith_short_12","alias_value":"6OW6THKG5VIE","created_at":"2026-07-05T08:31:16Z"},{"alias_kind":"pith_short_16","alias_value":"6OW6THKG5VIE63IH","created_at":"2026-07-05T08:31:16Z"},{"alias_kind":"pith_short_8","alias_value":"6OW6THKG","created_at":"2026-07-05T08:31:16Z"}],"graph_snapshots":[{"event_id":"sha256:801b507d57abff79a32a88a371407a7c01e13a27eaff7d8b1b7cd5bd244257b4","target":"graph","created_at":"2026-07-05T08:31:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.00343/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The deployment of Large Language Models (LLMs) in real-world applications presents both opportunities and challenges, particularly in multilingual and code-mixed communication settings. This research evaluates the performance of seven leading LLMs in sentiment analysis on a dataset derived from multilingual and code-mixed WhatsApp chats, including Swahili, English and Sheng. Our evaluation includes both quantitative analysis using metrics like F1 score and qualitative assessment of LLMs' explanations for their predictions. We find that, while Mistral-7b and Mixtral-8x7b achieved high F1 scores","authors_text":"Jacki O'Neill, Jindong Wang, Kalika Bali, Keshet Ronen, Millicent Ochieng, Sunayana Sitaram, Varun Gumma, Vishrav Chaudhary","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-01T07:36:59Z","title":"Beyond Metrics: Evaluating LLMs' Effectiveness in Culturally Nuanced, Low-Resource Real-World Scenarios"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.00343","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:84773ca53dab07cfda2326c21d26cc6ffceff08ca4829b5b0365b4a533ff7ee4","target":"record","created_at":"2026-07-05T08:31:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"19c122f324c440c25fcb35f37d6abd4d4a6966e854b0ad49d06de69595384851","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-01T07:36:59Z","title_canon_sha256":"d35d622bc47f7af514a31a50a747ad4e623b24e04ecd6f3b4512f2e9253a923e"},"schema_version":"1.0","source":{"id":"2406.00343","kind":"arxiv","version":2}},"canonical_sha256":"f3ade99d46ed504f6d079d39e52b1e3031a77699e2f4c91cc3940ee37370cba5","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f3ade99d46ed504f6d079d39e52b1e3031a77699e2f4c91cc3940ee37370cba5","first_computed_at":"2026-07-05T08:31:16.012586Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:31:16.012586Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"fOWU4/Ze6k+/EPw+Ch1g/852oJcTjhLegSddGEj4kZbPtYASP0IhX7PDAhm9CshxgPQQFYC5XFkumnqGk3EUAg==","signature_status":"signed_v1","signed_at":"2026-07-05T08:31:16.012990Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.00343","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:84773ca53dab07cfda2326c21d26cc6ffceff08ca4829b5b0365b4a533ff7ee4","sha256:801b507d57abff79a32a88a371407a7c01e13a27eaff7d8b1b7cd5bd244257b4"],"state_sha256":"9cda147613fca3ff17166d513669ca5d68e430665bf4f435dda768775d1eec8a"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"rWY5YhaPSYy3PnRRrLa+F/ukjScz4rh83VQKf7DBoKxhkau+f5T67AFRRPCr4cXP88IqC2iGsp+XovbUu0bkBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T05:22:23.023860Z","bundle_sha256":"ed1a19bcb95707553b267a557bb2403c100b20770899bf0147cca28890b43cc4"}}