{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2018:XGMMISVGU2VSQBVBC6YEXWAYMA","short_pith_number":"pith:XGMMISVG","schema_version":"1.0","canonical_sha256":"b998c44aa6a6ab2806a117b04bd818601e83b4ddd5302082a74154367f11e693","source":{"kind":"arxiv","id":"1807.06998","version":1},"attestation_state":"computed","paper":{"title":"Is it worth it? Budget-related evaluation metrics for model selection","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Filip Klubi\\v{c}ka, Giancarlo D. Salton, John D. Kelleher","submitted_at":"2018-07-18T15:37:58Z","abstract_excerpt":"Creating a linguistic resource is often done by using a machine learning model that filters the content that goes through to a human annotator, before going into the final resource. However, budgets are often limited, and the amount of available data exceeds the amount of affordable annotation. In order to optimize the benefit from the invested human work, we argue that deciding on which model one should employ depends not only on generalized evaluation metrics such as F-score, but also on the gain metric. Because the model with the highest F-score may not necessarily have the best sequencing "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"1807.06998","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2018-07-18T15:37:58Z","cross_cats_sorted":[],"title_canon_sha256":"e1dcab158bc8f343d33b0f22dcad3e5d99123df35f12a6b475571ef642d2d9d3","abstract_canon_sha256":"3ba0f06005689b2881f3d5e9173c7c5f53853b5a3813040f52a45c16eacf744b"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T00:10:25.129557Z","signature_b64":"a+YZ5bIhYNECqUZllDlPKX2EQHom3Mvgm3K9o3lj+W8oGanQ2+q7PZvequKWD50aPgupzkSOeNQd+SeRZuXJCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b998c44aa6a6ab2806a117b04bd818601e83b4ddd5302082a74154367f11e693","last_reissued_at":"2026-05-18T00:10:25.128866Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T00:10:25.128866Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Is it worth it? Budget-related evaluation metrics for model selection","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Filip Klubi\\v{c}ka, Giancarlo D. Salton, John D. Kelleher","submitted_at":"2018-07-18T15:37:58Z","abstract_excerpt":"Creating a linguistic resource is often done by using a machine learning model that filters the content that goes through to a human annotator, before going into the final resource. However, budgets are often limited, and the amount of available data exceeds the amount of affordable annotation. In order to optimize the benefit from the invested human work, we argue that deciding on which model one should employ depends not only on generalized evaluation metrics such as F-score, but also on the gain metric. Because the model with the highest F-score may not necessarily have the best sequencing "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1807.06998","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"1807.06998","created_at":"2026-05-18T00:10:25.128993+00:00"},{"alias_kind":"arxiv_version","alias_value":"1807.06998v1","created_at":"2026-05-18T00:10:25.128993+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1807.06998","created_at":"2026-05-18T00:10:25.128993+00:00"},{"alias_kind":"pith_short_12","alias_value":"XGMMISVGU2VS","created_at":"2026-05-18T12:33:01.666342+00:00"},{"alias_kind":"pith_short_16","alias_value":"XGMMISVGU2VSQBVB","created_at":"2026-05-18T12:33:01.666342+00:00"},{"alias_kind":"pith_short_8","alias_value":"XGMMISVG","created_at":"2026-05-18T12:33:01.666342+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/XGMMISVGU2VSQBVBC6YEXWAYMA","json":"https://pith.science/pith/XGMMISVGU2VSQBVBC6YEXWAYMA.json","graph_json":"https://pith.science/api/pith-number/XGMMISVGU2VSQBVBC6YEXWAYMA/graph.json","events_json":"https://pith.science/api/pith-number/XGMMISVGU2VSQBVBC6YEXWAYMA/events.json","paper":"https://pith.science/paper/XGMMISVG"},"agent_actions":{"view_html":"https://pith.science/pith/XGMMISVGU2VSQBVBC6YEXWAYMA","download_json":"https://pith.science/pith/XGMMISVGU2VSQBVBC6YEXWAYMA.json","view_paper":"https://pith.science/paper/XGMMISVG","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=1807.06998&json=true","fetch_graph":"https://pith.science/api/pith-number/XGMMISVGU2VSQBVBC6YEXWAYMA/graph.json","fetch_events":"https://pith.science/api/pith-number/XGMMISVGU2VSQBVBC6YEXWAYMA/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/XGMMISVGU2VSQBVBC6YEXWAYMA/action/timestamp_anchor","attest_storage":"https://pith.science/pith/XGMMISVGU2VSQBVBC6YEXWAYMA/action/storage_attestation","attest_author":"https://pith.science/pith/XGMMISVGU2VSQBVBC6YEXWAYMA/action/author_attestation","sign_citation":"https://pith.science/pith/XGMMISVGU2VSQBVBC6YEXWAYMA/action/citation_signature","submit_replication":"https://pith.science/pith/XGMMISVGU2VSQBVBC6YEXWAYMA/action/replication_record"}},"created_at":"2026-05-18T00:10:25.128993+00:00","updated_at":"2026-05-18T00:10:25.128993+00:00"}