{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:EGRZEKHAJLS2LUEOYTTDAK5J74","short_pith_number":"pith:EGRZEKHA","schema_version":"1.0","canonical_sha256":"21a39228e04ae5a5d08ec4e6302ba9ff37a71ca4ae1d962a648405a56b4bdd6e","source":{"kind":"arxiv","id":"2401.03831","version":1},"attestation_state":"computed","paper":{"title":"We Need to Talk About Classification Evaluation Metrics in NLP","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Emilio Monti, Lo\\\"ic Barrault, Nikolaos Aletras, Peter Vickers","submitted_at":"2024-01-08T11:40:48Z","abstract_excerpt":"In Natural Language Processing (NLP) classification tasks such as topic categorisation and sentiment analysis, model generalizability is generally measured with standard metrics such as Accuracy, F-Measure, or AUC-ROC. The diversity of metrics, and the arbitrariness of their application suggest that there is no agreement within NLP on a single best metric to use. This lack suggests there has not been sufficient examination of the underlying heuristics which each metric encodes. To address this we compare several standard classification metrics with more 'exotic' metrics and demonstrate that a "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2401.03831","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-01-08T11:40:48Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"7fdbfb54d777ec80f141b68798963c33f1c3f6718e5e58e15c9f2cc28e570258","abstract_canon_sha256":"53bdeb8cb7906c75e769dcbf04b7fef7e303d019e45663dfd62857bf226814fc"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:31:14.319801Z","signature_b64":"9gFzBmoCVeH+VJYw2UnOvTcdhRUV3S/exyjhIw7YOXWb0b0E9CDOF+ZsGtOfE2RYh8Orh4U8LTEFwzuTSeZCBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"21a39228e04ae5a5d08ec4e6302ba9ff37a71ca4ae1d962a648405a56b4bdd6e","last_reissued_at":"2026-07-05T07:31:14.319379Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:31:14.319379Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"We Need to Talk About Classification Evaluation Metrics in NLP","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Emilio Monti, Lo\\\"ic Barrault, Nikolaos Aletras, Peter Vickers","submitted_at":"2024-01-08T11:40:48Z","abstract_excerpt":"In Natural Language Processing (NLP) classification tasks such as topic categorisation and sentiment analysis, model generalizability is generally measured with standard metrics such as Accuracy, F-Measure, or AUC-ROC. The diversity of metrics, and the arbitrariness of their application suggest that there is no agreement within NLP on a single best metric to use. This lack suggests there has not been sufficient examination of the underlying heuristics which each metric encodes. To address this we compare several standard classification metrics with more 'exotic' metrics and demonstrate that a "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2401.03831","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2401.03831/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2401.03831","created_at":"2026-07-05T07:31:14.319436+00:00"},{"alias_kind":"arxiv_version","alias_value":"2401.03831v1","created_at":"2026-07-05T07:31:14.319436+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2401.03831","created_at":"2026-07-05T07:31:14.319436+00:00"},{"alias_kind":"pith_short_12","alias_value":"EGRZEKHAJLS2","created_at":"2026-07-05T07:31:14.319436+00:00"},{"alias_kind":"pith_short_16","alias_value":"EGRZEKHAJLS2LUEO","created_at":"2026-07-05T07:31:14.319436+00:00"},{"alias_kind":"pith_short_8","alias_value":"EGRZEKHA","created_at":"2026-07-05T07:31:14.319436+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/EGRZEKHAJLS2LUEOYTTDAK5J74","json":"https://pith.science/pith/EGRZEKHAJLS2LUEOYTTDAK5J74.json","graph_json":"https://pith.science/api/pith-number/EGRZEKHAJLS2LUEOYTTDAK5J74/graph.json","events_json":"https://pith.science/api/pith-number/EGRZEKHAJLS2LUEOYTTDAK5J74/events.json","paper":"https://pith.science/paper/EGRZEKHA"},"agent_actions":{"view_html":"https://pith.science/pith/EGRZEKHAJLS2LUEOYTTDAK5J74","download_json":"https://pith.science/pith/EGRZEKHAJLS2LUEOYTTDAK5J74.json","view_paper":"https://pith.science/paper/EGRZEKHA","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2401.03831&json=true","fetch_graph":"https://pith.science/api/pith-number/EGRZEKHAJLS2LUEOYTTDAK5J74/graph.json","fetch_events":"https://pith.science/api/pith-number/EGRZEKHAJLS2LUEOYTTDAK5J74/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/EGRZEKHAJLS2LUEOYTTDAK5J74/action/timestamp_anchor","attest_storage":"https://pith.science/pith/EGRZEKHAJLS2LUEOYTTDAK5J74/action/storage_attestation","attest_author":"https://pith.science/pith/EGRZEKHAJLS2LUEOYTTDAK5J74/action/author_attestation","sign_citation":"https://pith.science/pith/EGRZEKHAJLS2LUEOYTTDAK5J74/action/citation_signature","submit_replication":"https://pith.science/pith/EGRZEKHAJLS2LUEOYTTDAK5J74/action/replication_record"}},"created_at":"2026-07-05T07:31:14.319436+00:00","updated_at":"2026-07-05T07:31:14.319436+00:00"}