{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:AUOGDFJGE4NSFNLY4EFN4F63UA","short_pith_number":"pith:AUOGDFJG","canonical_record":{"source":{"id":"2412.01186","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-02T06:42:51Z","cross_cats_sorted":[],"title_canon_sha256":"34575dc654072aee49aece6bbada905e3609bcc4e95ccda1e50df1253cf657dc","abstract_canon_sha256":"43e7c77d6f31df52db9363ac9a8c7f5f7d695beb1602b69fb4bc53e14389aeea"},"schema_version":"1.0"},"canonical_sha256":"051c619526271b22b578e10ade17dba01b155d58b3161916422d4a6fe3d3cb9b","source":{"kind":"arxiv","id":"2412.01186","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.01186","created_at":"2026-07-05T09:42:55Z"},{"alias_kind":"arxiv_version","alias_value":"2412.01186v1","created_at":"2026-07-05T09:42:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.01186","created_at":"2026-07-05T09:42:55Z"},{"alias_kind":"pith_short_12","alias_value":"AUOGDFJGE4NS","created_at":"2026-07-05T09:42:55Z"},{"alias_kind":"pith_short_16","alias_value":"AUOGDFJGE4NSFNLY","created_at":"2026-07-05T09:42:55Z"},{"alias_kind":"pith_short_8","alias_value":"AUOGDFJG","created_at":"2026-07-05T09:42:55Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:AUOGDFJGE4NSFNLY4EFN4F63UA","target":"record","payload":{"canonical_record":{"source":{"id":"2412.01186","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-02T06:42:51Z","cross_cats_sorted":[],"title_canon_sha256":"34575dc654072aee49aece6bbada905e3609bcc4e95ccda1e50df1253cf657dc","abstract_canon_sha256":"43e7c77d6f31df52db9363ac9a8c7f5f7d695beb1602b69fb4bc53e14389aeea"},"schema_version":"1.0"},"canonical_sha256":"051c619526271b22b578e10ade17dba01b155d58b3161916422d4a6fe3d3cb9b","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:42:55.520550Z","signature_b64":"q1jJW3umOyn+sl6gpHOWRo++8FS3fb1aPQtojEVbo1ndAqYDBZVlv7WoNiT8ge1P1ssVpezBHHgEvB6qdvkNAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"051c619526271b22b578e10ade17dba01b155d58b3161916422d4a6fe3d3cb9b","last_reissued_at":"2026-07-05T09:42:55.520058Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:42:55.520058Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2412.01186","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:42:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"RlwUDCGzlvSGZg/MvhKErnXAC6DoiD+rupBpyRTUJGeRWVXdYAwyOr5KPhoT3HyXsnewRiC8/kP2uBFXqh9VBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T14:50:50.526260Z"},"content_sha256":"d1866949aa6dd6c6ff1b9c112f8768608d78dca5b705aff9a324652dbe211a11","schema_version":"1.0","event_id":"sha256:d1866949aa6dd6c6ff1b9c112f8768608d78dca5b705aff9a324652dbe211a11"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:AUOGDFJGE4NSFNLY4EFN4F63UA","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"SailCompass: Towards Reproducible and Robust Evaluation for Southeast Asian Languages","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Guangtao Zeng, Jia Guo, Longxu Dou, Qian Liu, Stanley Kok, Wei Lu","submitted_at":"2024-12-02T06:42:51Z","abstract_excerpt":"In this paper, we introduce SailCompass, a reproducible and robust evaluation benchmark for assessing Large Language Models (LLMs) on Southeast Asian Languages (SEA). SailCompass encompasses three main SEA languages, eight primary tasks including 14 datasets covering three task types (generation, multiple-choice questions, and classification). To improve the robustness of the evaluation approach, we explore different prompt configurations for multiple-choice questions and leverage calibrations to improve the faithfulness of classification tasks. With SailCompass, we derive the following findin"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.01186","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2412.01186/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:42:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"kFzD3+lCk/CjtmVQsvspzt55IxJ/RkwPrbmXV7pxVivztbVhzsFuno2eHpfWX3frCnIo8bx5uLX8HaEwcQi7Bg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T14:50:50.526642Z"},"content_sha256":"d0242dcfa33a2147edb9aad639047911e35d512fadd8576db5901abee5c7cd5a","schema_version":"1.0","event_id":"sha256:d0242dcfa33a2147edb9aad639047911e35d512fadd8576db5901abee5c7cd5a"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/AUOGDFJGE4NSFNLY4EFN4F63UA/bundle.json","state_url":"https://pith.science/pith/AUOGDFJGE4NSFNLY4EFN4F63UA/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/AUOGDFJGE4NSFNLY4EFN4F63UA/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T14:50:50Z","links":{"resolver":"https://pith.science/pith/AUOGDFJGE4NSFNLY4EFN4F63UA","bundle":"https://pith.science/pith/AUOGDFJGE4NSFNLY4EFN4F63UA/bundle.json","state":"https://pith.science/pith/AUOGDFJGE4NSFNLY4EFN4F63UA/state.json","well_known_bundle":"https://pith.science/.well-known/pith/AUOGDFJGE4NSFNLY4EFN4F63UA/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:AUOGDFJGE4NSFNLY4EFN4F63UA","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"43e7c77d6f31df52db9363ac9a8c7f5f7d695beb1602b69fb4bc53e14389aeea","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-02T06:42:51Z","title_canon_sha256":"34575dc654072aee49aece6bbada905e3609bcc4e95ccda1e50df1253cf657dc"},"schema_version":"1.0","source":{"id":"2412.01186","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.01186","created_at":"2026-07-05T09:42:55Z"},{"alias_kind":"arxiv_version","alias_value":"2412.01186v1","created_at":"2026-07-05T09:42:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.01186","created_at":"2026-07-05T09:42:55Z"},{"alias_kind":"pith_short_12","alias_value":"AUOGDFJGE4NS","created_at":"2026-07-05T09:42:55Z"},{"alias_kind":"pith_short_16","alias_value":"AUOGDFJGE4NSFNLY","created_at":"2026-07-05T09:42:55Z"},{"alias_kind":"pith_short_8","alias_value":"AUOGDFJG","created_at":"2026-07-05T09:42:55Z"}],"graph_snapshots":[{"event_id":"sha256:d0242dcfa33a2147edb9aad639047911e35d512fadd8576db5901abee5c7cd5a","target":"graph","created_at":"2026-07-05T09:42:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2412.01186/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In this paper, we introduce SailCompass, a reproducible and robust evaluation benchmark for assessing Large Language Models (LLMs) on Southeast Asian Languages (SEA). SailCompass encompasses three main SEA languages, eight primary tasks including 14 datasets covering three task types (generation, multiple-choice questions, and classification). To improve the robustness of the evaluation approach, we explore different prompt configurations for multiple-choice questions and leverage calibrations to improve the faithfulness of classification tasks. With SailCompass, we derive the following findin","authors_text":"Guangtao Zeng, Jia Guo, Longxu Dou, Qian Liu, Stanley Kok, Wei Lu","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-02T06:42:51Z","title":"SailCompass: Towards Reproducible and Robust Evaluation for Southeast Asian Languages"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.01186","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:d1866949aa6dd6c6ff1b9c112f8768608d78dca5b705aff9a324652dbe211a11","target":"record","created_at":"2026-07-05T09:42:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"43e7c77d6f31df52db9363ac9a8c7f5f7d695beb1602b69fb4bc53e14389aeea","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-02T06:42:51Z","title_canon_sha256":"34575dc654072aee49aece6bbada905e3609bcc4e95ccda1e50df1253cf657dc"},"schema_version":"1.0","source":{"id":"2412.01186","kind":"arxiv","version":1}},"canonical_sha256":"051c619526271b22b578e10ade17dba01b155d58b3161916422d4a6fe3d3cb9b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"051c619526271b22b578e10ade17dba01b155d58b3161916422d4a6fe3d3cb9b","first_computed_at":"2026-07-05T09:42:55.520058Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:42:55.520058Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"q1jJW3umOyn+sl6gpHOWRo++8FS3fb1aPQtojEVbo1ndAqYDBZVlv7WoNiT8ge1P1ssVpezBHHgEvB6qdvkNAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T09:42:55.520550Z","signed_message":"canonical_sha256_bytes"},"source_id":"2412.01186","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:d1866949aa6dd6c6ff1b9c112f8768608d78dca5b705aff9a324652dbe211a11","sha256:d0242dcfa33a2147edb9aad639047911e35d512fadd8576db5901abee5c7cd5a"],"state_sha256":"5316878c1b15aeb5826ee1e31e60ae2ed77321e981b6ad3ca522759d3d1d152c"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CyNaIeiQpeFU8keGgLDIELjntWrC/t1rfGPxwAPfdNyAjOMhLWPTv5nP1VqnWYzTSfuXvloT55NDcInxkYpSBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T14:50:50.528550Z","bundle_sha256":"0be3fa65aefa0509ae932bc1e0c0c3327ffff8233aa925a351a93a882622b2c0"}}