{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:4ISME73ACT7CD4YBWNI3Q5DGJC","short_pith_number":"pith:4ISME73A","canonical_record":{"source":{"id":"2305.10284","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-17T15:20:31Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"fdff8b921dc68956d80aa0e99237a78967521f5e830b334f5e1316a2d2ebce89","abstract_canon_sha256":"dee0ffce63415bf089d2eebb772742cac7212fc1d42f3640f8d8a2993358ac01"},"schema_version":"1.0"},"canonical_sha256":"e224c27f6014fe21f301b351b87466488375f151eb7d98b3a730872e41ec69ba","source":{"kind":"arxiv","id":"2305.10284","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.10284","created_at":"2026-07-05T06:11:12Z"},{"alias_kind":"arxiv_version","alias_value":"2305.10284v1","created_at":"2026-07-05T06:11:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.10284","created_at":"2026-07-05T06:11:12Z"},{"alias_kind":"pith_short_12","alias_value":"4ISME73ACT7C","created_at":"2026-07-05T06:11:12Z"},{"alias_kind":"pith_short_16","alias_value":"4ISME73ACT7CD4YB","created_at":"2026-07-05T06:11:12Z"},{"alias_kind":"pith_short_8","alias_value":"4ISME73A","created_at":"2026-07-05T06:11:12Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:4ISME73ACT7CD4YBWNI3Q5DGJC","target":"record","payload":{"canonical_record":{"source":{"id":"2305.10284","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-17T15:20:31Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"fdff8b921dc68956d80aa0e99237a78967521f5e830b334f5e1316a2d2ebce89","abstract_canon_sha256":"dee0ffce63415bf089d2eebb772742cac7212fc1d42f3640f8d8a2993358ac01"},"schema_version":"1.0"},"canonical_sha256":"e224c27f6014fe21f301b351b87466488375f151eb7d98b3a730872e41ec69ba","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:11:12.141406Z","signature_b64":"9VIDNzX1es7TPLnfmFmQ88uTk1v/p5WINnAme9wtibsA6s+d3JqxvXNr8q2+M5TkMsk+KyHR0ZdvenLzG3N2AQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"e224c27f6014fe21f301b351b87466488375f151eb7d98b3a730872e41ec69ba","last_reissued_at":"2026-07-05T06:11:12.141009Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:11:12.141009Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2305.10284","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:11:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"60NCIs+0E8LkVp4DKkWaSvUYGcl8qUWSMBVZNx3NATHBqbWw8TuGsAnBmbzWpr0ESTScPrHIACRHd4hwQ1OSBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T01:48:42.865072Z"},"content_sha256":"dc0dd061a2c1e58188f79e77aff49ad8dc19bce28aa276cc4951fa86e77ab2e5","schema_version":"1.0","event_id":"sha256:dc0dd061a2c1e58188f79e77aff49ad8dc19bce28aa276cc4951fa86e77ab2e5"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:4ISME73ACT7CD4YBWNI3Q5DGJC","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Towards More Robust NLP System Evaluation: Handling Missing Scores in Benchmarks","license":"http://creativecommons.org/publicdomain/zero/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Anas Himmi, Ekhine Irurozki, Nathan Noiry, Pierre Colombo, Stephan Clemencon","submitted_at":"2023-05-17T15:20:31Z","abstract_excerpt":"The evaluation of natural language processing (NLP) systems is crucial for advancing the field, but current benchmarking approaches often assume that all systems have scores available for all tasks, which is not always practical. In reality, several factors such as the cost of running baseline, private systems, computational limitations, or incomplete data may prevent some systems from being evaluated on entire tasks. This paper formalize an existing problem in NLP research: benchmarking when some systems scores are missing on the task, and proposes a novel approach to address it. Our method u"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.10284","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.10284/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:11:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"9ieUFuLRBIjFm22Hao4yas4ts5iwpk3ynLI2MuFxmLBENapZs731L01VhQmCCrTSIh7tyor1bwhhaD8ZASqgDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T01:48:42.865583Z"},"content_sha256":"b0dcd30a8c62a364a6b371d5c78d6ab3271443e3979dc8b85a6c793124b6a871","schema_version":"1.0","event_id":"sha256:b0dcd30a8c62a364a6b371d5c78d6ab3271443e3979dc8b85a6c793124b6a871"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/4ISME73ACT7CD4YBWNI3Q5DGJC/bundle.json","state_url":"https://pith.science/pith/4ISME73ACT7CD4YBWNI3Q5DGJC/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/4ISME73ACT7CD4YBWNI3Q5DGJC/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T01:48:42Z","links":{"resolver":"https://pith.science/pith/4ISME73ACT7CD4YBWNI3Q5DGJC","bundle":"https://pith.science/pith/4ISME73ACT7CD4YBWNI3Q5DGJC/bundle.json","state":"https://pith.science/pith/4ISME73ACT7CD4YBWNI3Q5DGJC/state.json","well_known_bundle":"https://pith.science/.well-known/pith/4ISME73ACT7CD4YBWNI3Q5DGJC/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:4ISME73ACT7CD4YBWNI3Q5DGJC","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"dee0ffce63415bf089d2eebb772742cac7212fc1d42f3640f8d8a2993358ac01","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-17T15:20:31Z","title_canon_sha256":"fdff8b921dc68956d80aa0e99237a78967521f5e830b334f5e1316a2d2ebce89"},"schema_version":"1.0","source":{"id":"2305.10284","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.10284","created_at":"2026-07-05T06:11:12Z"},{"alias_kind":"arxiv_version","alias_value":"2305.10284v1","created_at":"2026-07-05T06:11:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.10284","created_at":"2026-07-05T06:11:12Z"},{"alias_kind":"pith_short_12","alias_value":"4ISME73ACT7C","created_at":"2026-07-05T06:11:12Z"},{"alias_kind":"pith_short_16","alias_value":"4ISME73ACT7CD4YB","created_at":"2026-07-05T06:11:12Z"},{"alias_kind":"pith_short_8","alias_value":"4ISME73A","created_at":"2026-07-05T06:11:12Z"}],"graph_snapshots":[{"event_id":"sha256:b0dcd30a8c62a364a6b371d5c78d6ab3271443e3979dc8b85a6c793124b6a871","target":"graph","created_at":"2026-07-05T06:11:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2305.10284/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The evaluation of natural language processing (NLP) systems is crucial for advancing the field, but current benchmarking approaches often assume that all systems have scores available for all tasks, which is not always practical. In reality, several factors such as the cost of running baseline, private systems, computational limitations, or incomplete data may prevent some systems from being evaluated on entire tasks. This paper formalize an existing problem in NLP research: benchmarking when some systems scores are missing on the task, and proposes a novel approach to address it. Our method u","authors_text":"Anas Himmi, Ekhine Irurozki, Nathan Noiry, Pierre Colombo, Stephan Clemencon","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-17T15:20:31Z","title":"Towards More Robust NLP System Evaluation: Handling Missing Scores in Benchmarks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.10284","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:dc0dd061a2c1e58188f79e77aff49ad8dc19bce28aa276cc4951fa86e77ab2e5","target":"record","created_at":"2026-07-05T06:11:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"dee0ffce63415bf089d2eebb772742cac7212fc1d42f3640f8d8a2993358ac01","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-17T15:20:31Z","title_canon_sha256":"fdff8b921dc68956d80aa0e99237a78967521f5e830b334f5e1316a2d2ebce89"},"schema_version":"1.0","source":{"id":"2305.10284","kind":"arxiv","version":1}},"canonical_sha256":"e224c27f6014fe21f301b351b87466488375f151eb7d98b3a730872e41ec69ba","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"e224c27f6014fe21f301b351b87466488375f151eb7d98b3a730872e41ec69ba","first_computed_at":"2026-07-05T06:11:12.141009Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:11:12.141009Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"9VIDNzX1es7TPLnfmFmQ88uTk1v/p5WINnAme9wtibsA6s+d3JqxvXNr8q2+M5TkMsk+KyHR0ZdvenLzG3N2AQ==","signature_status":"signed_v1","signed_at":"2026-07-05T06:11:12.141406Z","signed_message":"canonical_sha256_bytes"},"source_id":"2305.10284","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:dc0dd061a2c1e58188f79e77aff49ad8dc19bce28aa276cc4951fa86e77ab2e5","sha256:b0dcd30a8c62a364a6b371d5c78d6ab3271443e3979dc8b85a6c793124b6a871"],"state_sha256":"826fcda40f681d7ed7c988229f0d51b152e05e720c4a6a59378c3f93e08072a7"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Me5wnlllvF2EOOrVTn5DE/W7+sPGROlilxDx0hFVe8ZtlYAuQ87lyaS/xoHREAZECC8PZa0ZS/4To1PnRng9Dw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T01:48:42.869371Z","bundle_sha256":"8ce0529ba16f4e09670dc0d76932a025967cb9f5931d558ca324995f3127b745"}}