{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:SZ2IGQWQDHJSLKBI4HQVTWKZXE","short_pith_number":"pith:SZ2IGQWQ","canonical_record":{"source":{"id":"2504.09714","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-04-13T20:45:49Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"5a99dcbab9ad7d485963515e097d651a39653cb5909bac0f04cd0f07a3b36824","abstract_canon_sha256":"a42d2bf541f5429f66a009bce1341805bad4d97093219e4395c2ea7b7eac7582"},"schema_version":"1.0"},"canonical_sha256":"96748342d019d325a828e1e159d959b9291d8d14d56204ea01b3beab300a3256","source":{"kind":"arxiv","id":"2504.09714","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2504.09714","created_at":"2026-07-05T10:54:16Z"},{"alias_kind":"arxiv_version","alias_value":"2504.09714v2","created_at":"2026-07-05T10:54:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.09714","created_at":"2026-07-05T10:54:16Z"},{"alias_kind":"pith_short_12","alias_value":"SZ2IGQWQDHJS","created_at":"2026-07-05T10:54:16Z"},{"alias_kind":"pith_short_16","alias_value":"SZ2IGQWQDHJSLKBI","created_at":"2026-07-05T10:54:16Z"},{"alias_kind":"pith_short_8","alias_value":"SZ2IGQWQ","created_at":"2026-07-05T10:54:16Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:SZ2IGQWQDHJSLKBI4HQVTWKZXE","target":"record","payload":{"canonical_record":{"source":{"id":"2504.09714","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-04-13T20:45:49Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"5a99dcbab9ad7d485963515e097d651a39653cb5909bac0f04cd0f07a3b36824","abstract_canon_sha256":"a42d2bf541f5429f66a009bce1341805bad4d97093219e4395c2ea7b7eac7582"},"schema_version":"1.0"},"canonical_sha256":"96748342d019d325a828e1e159d959b9291d8d14d56204ea01b3beab300a3256","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:54:16.184185Z","signature_b64":"14mPzG76BcI7pn/2OSD4G6OkrNdHpcJ+E/HiFndFG0vM72QmblLys3XrSQiKPM8RhWhAf/Q4twgEslu9hZCJAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"96748342d019d325a828e1e159d959b9291d8d14d56204ea01b3beab300a3256","last_reissued_at":"2026-07-05T10:54:16.183678Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:54:16.183678Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2504.09714","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:54:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"j4snfaOxBHJqUl8dqczL7P8U8D3dFHjG9BPxxzepYByMYMHND5OkNzwSO8xoXFXlNFnMxXzzwjoxuomobKyOAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T13:05:18.516029Z"},"content_sha256":"eefcc4b54385a4ea48112940fd05ef86ef02f4760db723c3d7dbd8d62b3a336b","schema_version":"1.0","event_id":"sha256:eefcc4b54385a4ea48112940fd05ef86ef02f4760db723c3d7dbd8d62b3a336b"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:SZ2IGQWQDHJSLKBI4HQVTWKZXE","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Evaluating the Quality of Benchmark Datasets for Low-Resource Languages: A Case Study on Turkish","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Abdullah Topraksoy, Ahmet Kaan Sever, Ay\\c{s}e Aysu Cengiz, B\\\"u\\c{s}ra Tufan, Burak Aytan, Cagri Toraman, Elif Ecem \\\"Um\\\"utl\\\"u, Esra Dar{\\i}c{\\i}, Naime \\c{S}eyma Erdem","submitted_at":"2025-04-13T20:45:49Z","abstract_excerpt":"The reliance on translated or adapted datasets from English or multilingual resources introduces challenges regarding linguistic and cultural suitability. This study addresses the need for robust and culturally appropriate benchmarks by evaluating the quality of 17 commonly used Turkish benchmark datasets. Using a comprehensive framework that assesses six criteria, both human and LLM-judge annotators provide detailed evaluations to identify dataset strengths and shortcomings.\n  Our results reveal that 70% of the benchmark datasets fail to meet our heuristic quality standards. The correctness o"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2504.09714","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2504.09714/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:54:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"oohlaKFhcBQ4WKcNGpZ6yU0myxE+4E27/mvUPTEOoGEkAsJje+AUkjT+Bzy5RfouIfEDazEVQWMV/JO35RkMCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T13:05:18.516549Z"},"content_sha256":"4e7589b5c679a1bb369f1619f0c29e50b7ccd067409ff64ba8efca7d43553538","schema_version":"1.0","event_id":"sha256:4e7589b5c679a1bb369f1619f0c29e50b7ccd067409ff64ba8efca7d43553538"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/SZ2IGQWQDHJSLKBI4HQVTWKZXE/bundle.json","state_url":"https://pith.science/pith/SZ2IGQWQDHJSLKBI4HQVTWKZXE/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/SZ2IGQWQDHJSLKBI4HQVTWKZXE/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-06T13:05:18Z","links":{"resolver":"https://pith.science/pith/SZ2IGQWQDHJSLKBI4HQVTWKZXE","bundle":"https://pith.science/pith/SZ2IGQWQDHJSLKBI4HQVTWKZXE/bundle.json","state":"https://pith.science/pith/SZ2IGQWQDHJSLKBI4HQVTWKZXE/state.json","well_known_bundle":"https://pith.science/.well-known/pith/SZ2IGQWQDHJSLKBI4HQVTWKZXE/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:SZ2IGQWQDHJSLKBI4HQVTWKZXE","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"a42d2bf541f5429f66a009bce1341805bad4d97093219e4395c2ea7b7eac7582","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-04-13T20:45:49Z","title_canon_sha256":"5a99dcbab9ad7d485963515e097d651a39653cb5909bac0f04cd0f07a3b36824"},"schema_version":"1.0","source":{"id":"2504.09714","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2504.09714","created_at":"2026-07-05T10:54:16Z"},{"alias_kind":"arxiv_version","alias_value":"2504.09714v2","created_at":"2026-07-05T10:54:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.09714","created_at":"2026-07-05T10:54:16Z"},{"alias_kind":"pith_short_12","alias_value":"SZ2IGQWQDHJS","created_at":"2026-07-05T10:54:16Z"},{"alias_kind":"pith_short_16","alias_value":"SZ2IGQWQDHJSLKBI","created_at":"2026-07-05T10:54:16Z"},{"alias_kind":"pith_short_8","alias_value":"SZ2IGQWQ","created_at":"2026-07-05T10:54:16Z"}],"graph_snapshots":[{"event_id":"sha256:4e7589b5c679a1bb369f1619f0c29e50b7ccd067409ff64ba8efca7d43553538","target":"graph","created_at":"2026-07-05T10:54:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2504.09714/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The reliance on translated or adapted datasets from English or multilingual resources introduces challenges regarding linguistic and cultural suitability. This study addresses the need for robust and culturally appropriate benchmarks by evaluating the quality of 17 commonly used Turkish benchmark datasets. Using a comprehensive framework that assesses six criteria, both human and LLM-judge annotators provide detailed evaluations to identify dataset strengths and shortcomings.\n  Our results reveal that 70% of the benchmark datasets fail to meet our heuristic quality standards. The correctness o","authors_text":"Abdullah Topraksoy, Ahmet Kaan Sever, Ay\\c{s}e Aysu Cengiz, B\\\"u\\c{s}ra Tufan, Burak Aytan, Cagri Toraman, Elif Ecem \\\"Um\\\"utl\\\"u, Esra Dar{\\i}c{\\i}, Naime \\c{S}eyma Erdem","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-04-13T20:45:49Z","title":"Evaluating the Quality of Benchmark Datasets for Low-Resource Languages: A Case Study on Turkish"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2504.09714","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:eefcc4b54385a4ea48112940fd05ef86ef02f4760db723c3d7dbd8d62b3a336b","target":"record","created_at":"2026-07-05T10:54:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"a42d2bf541f5429f66a009bce1341805bad4d97093219e4395c2ea7b7eac7582","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-04-13T20:45:49Z","title_canon_sha256":"5a99dcbab9ad7d485963515e097d651a39653cb5909bac0f04cd0f07a3b36824"},"schema_version":"1.0","source":{"id":"2504.09714","kind":"arxiv","version":2}},"canonical_sha256":"96748342d019d325a828e1e159d959b9291d8d14d56204ea01b3beab300a3256","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"96748342d019d325a828e1e159d959b9291d8d14d56204ea01b3beab300a3256","first_computed_at":"2026-07-05T10:54:16.183678Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:54:16.183678Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"14mPzG76BcI7pn/2OSD4G6OkrNdHpcJ+E/HiFndFG0vM72QmblLys3XrSQiKPM8RhWhAf/Q4twgEslu9hZCJAg==","signature_status":"signed_v1","signed_at":"2026-07-05T10:54:16.184185Z","signed_message":"canonical_sha256_bytes"},"source_id":"2504.09714","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:eefcc4b54385a4ea48112940fd05ef86ef02f4760db723c3d7dbd8d62b3a336b","sha256:4e7589b5c679a1bb369f1619f0c29e50b7ccd067409ff64ba8efca7d43553538"],"state_sha256":"1efefa0e4bcc27e5cf53617fcc4f2211b673ce803fedfdc2550c29bc316b5195"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"XAp9PCJPuzJh3zADOj2xdMQL8liTzn/qgLEfUU7tOvSWzCvZPskcOpqo3xCD99ZXIjYTrEGHakW4nvj8SbwvAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-06T13:05:18.520687Z","bundle_sha256":"c8a9a07183b80711c24ae31aa613aa2b0284ee93d980eab40a9e9a413cd119c1"}}