{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2018:XGMMISVGU2VSQBVBC6YEXWAYMA","short_pith_number":"pith:XGMMISVG","canonical_record":{"source":{"id":"1807.06998","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2018-07-18T15:37:58Z","cross_cats_sorted":[],"title_canon_sha256":"e1dcab158bc8f343d33b0f22dcad3e5d99123df35f12a6b475571ef642d2d9d3","abstract_canon_sha256":"3ba0f06005689b2881f3d5e9173c7c5f53853b5a3813040f52a45c16eacf744b"},"schema_version":"1.0"},"canonical_sha256":"b998c44aa6a6ab2806a117b04bd818601e83b4ddd5302082a74154367f11e693","source":{"kind":"arxiv","id":"1807.06998","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1807.06998","created_at":"2026-05-18T00:10:25Z"},{"alias_kind":"arxiv_version","alias_value":"1807.06998v1","created_at":"2026-05-18T00:10:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1807.06998","created_at":"2026-05-18T00:10:25Z"},{"alias_kind":"pith_short_12","alias_value":"XGMMISVGU2VS","created_at":"2026-05-18T12:33:01Z"},{"alias_kind":"pith_short_16","alias_value":"XGMMISVGU2VSQBVB","created_at":"2026-05-18T12:33:01Z"},{"alias_kind":"pith_short_8","alias_value":"XGMMISVG","created_at":"2026-05-18T12:33:01Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2018:XGMMISVGU2VSQBVBC6YEXWAYMA","target":"record","payload":{"canonical_record":{"source":{"id":"1807.06998","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2018-07-18T15:37:58Z","cross_cats_sorted":[],"title_canon_sha256":"e1dcab158bc8f343d33b0f22dcad3e5d99123df35f12a6b475571ef642d2d9d3","abstract_canon_sha256":"3ba0f06005689b2881f3d5e9173c7c5f53853b5a3813040f52a45c16eacf744b"},"schema_version":"1.0"},"canonical_sha256":"b998c44aa6a6ab2806a117b04bd818601e83b4ddd5302082a74154367f11e693","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T00:10:25.129557Z","signature_b64":"a+YZ5bIhYNECqUZllDlPKX2EQHom3Mvgm3K9o3lj+W8oGanQ2+q7PZvequKWD50aPgupzkSOeNQd+SeRZuXJCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b998c44aa6a6ab2806a117b04bd818601e83b4ddd5302082a74154367f11e693","last_reissued_at":"2026-05-18T00:10:25.128866Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T00:10:25.128866Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1807.06998","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:10:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"I9O7JGAtuxwbVK4nfk8JToARRF03bk/s+bih9CaGKi5s4etmD/+TFYvdsNxqDqjiAnstCfBpUwXmLjVe3reDAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-03T07:42:43.437995Z"},"content_sha256":"757d64a1d18d6def67b6a54f077061cd06498328be349400a20ed34cc98153c7","schema_version":"1.0","event_id":"sha256:757d64a1d18d6def67b6a54f077061cd06498328be349400a20ed34cc98153c7"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2018:XGMMISVGU2VSQBVBC6YEXWAYMA","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Is it worth it? Budget-related evaluation metrics for model selection","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Filip Klubi\\v{c}ka, Giancarlo D. Salton, John D. Kelleher","submitted_at":"2018-07-18T15:37:58Z","abstract_excerpt":"Creating a linguistic resource is often done by using a machine learning model that filters the content that goes through to a human annotator, before going into the final resource. However, budgets are often limited, and the amount of available data exceeds the amount of affordable annotation. In order to optimize the benefit from the invested human work, we argue that deciding on which model one should employ depends not only on generalized evaluation metrics such as F-score, but also on the gain metric. Because the model with the highest F-score may not necessarily have the best sequencing "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1807.06998","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:10:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"dr0I6Bbx8UdonjEiygMDbKoqgYKL9YjvfBTHzZudm7obYp9wafGK8Ey6ibA9ijM06FJvrtpOrYKCtZ+cxkamBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-03T07:42:43.438379Z"},"content_sha256":"b63a93982633cd0ee3d8240c483824775144b6c08eef6b4d1b9a058f42b5186d","schema_version":"1.0","event_id":"sha256:b63a93982633cd0ee3d8240c483824775144b6c08eef6b4d1b9a058f42b5186d"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/XGMMISVGU2VSQBVBC6YEXWAYMA/bundle.json","state_url":"https://pith.science/pith/XGMMISVGU2VSQBVBC6YEXWAYMA/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/XGMMISVGU2VSQBVBC6YEXWAYMA/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-06-03T07:42:43Z","links":{"resolver":"https://pith.science/pith/XGMMISVGU2VSQBVBC6YEXWAYMA","bundle":"https://pith.science/pith/XGMMISVGU2VSQBVBC6YEXWAYMA/bundle.json","state":"https://pith.science/pith/XGMMISVGU2VSQBVBC6YEXWAYMA/state.json","well_known_bundle":"https://pith.science/.well-known/pith/XGMMISVGU2VSQBVBC6YEXWAYMA/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2018:XGMMISVGU2VSQBVBC6YEXWAYMA","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"3ba0f06005689b2881f3d5e9173c7c5f53853b5a3813040f52a45c16eacf744b","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2018-07-18T15:37:58Z","title_canon_sha256":"e1dcab158bc8f343d33b0f22dcad3e5d99123df35f12a6b475571ef642d2d9d3"},"schema_version":"1.0","source":{"id":"1807.06998","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1807.06998","created_at":"2026-05-18T00:10:25Z"},{"alias_kind":"arxiv_version","alias_value":"1807.06998v1","created_at":"2026-05-18T00:10:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1807.06998","created_at":"2026-05-18T00:10:25Z"},{"alias_kind":"pith_short_12","alias_value":"XGMMISVGU2VS","created_at":"2026-05-18T12:33:01Z"},{"alias_kind":"pith_short_16","alias_value":"XGMMISVGU2VSQBVB","created_at":"2026-05-18T12:33:01Z"},{"alias_kind":"pith_short_8","alias_value":"XGMMISVG","created_at":"2026-05-18T12:33:01Z"}],"graph_snapshots":[{"event_id":"sha256:b63a93982633cd0ee3d8240c483824775144b6c08eef6b4d1b9a058f42b5186d","target":"graph","created_at":"2026-05-18T00:10:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"Creating a linguistic resource is often done by using a machine learning model that filters the content that goes through to a human annotator, before going into the final resource. However, budgets are often limited, and the amount of available data exceeds the amount of affordable annotation. In order to optimize the benefit from the invested human work, we argue that deciding on which model one should employ depends not only on generalized evaluation metrics such as F-score, but also on the gain metric. Because the model with the highest F-score may not necessarily have the best sequencing ","authors_text":"Filip Klubi\\v{c}ka, Giancarlo D. Salton, John D. Kelleher","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2018-07-18T15:37:58Z","title":"Is it worth it? Budget-related evaluation metrics for model selection"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1807.06998","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:757d64a1d18d6def67b6a54f077061cd06498328be349400a20ed34cc98153c7","target":"record","created_at":"2026-05-18T00:10:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"3ba0f06005689b2881f3d5e9173c7c5f53853b5a3813040f52a45c16eacf744b","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2018-07-18T15:37:58Z","title_canon_sha256":"e1dcab158bc8f343d33b0f22dcad3e5d99123df35f12a6b475571ef642d2d9d3"},"schema_version":"1.0","source":{"id":"1807.06998","kind":"arxiv","version":1}},"canonical_sha256":"b998c44aa6a6ab2806a117b04bd818601e83b4ddd5302082a74154367f11e693","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"b998c44aa6a6ab2806a117b04bd818601e83b4ddd5302082a74154367f11e693","first_computed_at":"2026-05-18T00:10:25.128866Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-18T00:10:25.128866Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"a+YZ5bIhYNECqUZllDlPKX2EQHom3Mvgm3K9o3lj+W8oGanQ2+q7PZvequKWD50aPgupzkSOeNQd+SeRZuXJCA==","signature_status":"signed_v1","signed_at":"2026-05-18T00:10:25.129557Z","signed_message":"canonical_sha256_bytes"},"source_id":"1807.06998","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:757d64a1d18d6def67b6a54f077061cd06498328be349400a20ed34cc98153c7","sha256:b63a93982633cd0ee3d8240c483824775144b6c08eef6b4d1b9a058f42b5186d"],"state_sha256":"71ce669d3a744fa4e49f8cf012442050188347266283c2f37af8e66979fcfc4f"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"iEkyLNNZgxKqjgnNNXuW0OBYCks+xpqs/KisaWoZJAwVANVusX7jZHOW5WUdXUvZESyCmN39ArxDIejjl+dQBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-06-03T07:42:43.440452Z","bundle_sha256":"ca1b26043a423a5bc5527510f355a89ee148939bea4c0e8df9f0ee53f7ffb63a"}}