{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:FHEZPXP5JC36QLGUWLNBEVJUMA","short_pith_number":"pith:FHEZPXP5","schema_version":"1.0","canonical_sha256":"29c997ddfd48b7e82cd4b2da125534600ed3ad3f8305af194393f9356ac2bd41","source":{"kind":"arxiv","id":"2406.15227","version":3},"attestation_state":"computed","paper":{"title":"A LLM-Based Ranking Method for the Evaluation of Automatic Counter-Narrative Generation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Aitor Soroa, Irune Zubiaga, Rodrigo Agerri","submitted_at":"2024-06-21T15:11:33Z","abstract_excerpt":"This paper proposes a novel approach to evaluate Counter Narrative (CN) generation using a Large Language Model (LLM) as an evaluator. We show that traditional automatic metrics correlate poorly with human judgements and fail to capture the nuanced relationship between generated CNs and human perception. To alleviate this, we introduce a model ranking pipeline based on pairwise comparisons of generated CNs from different models, organized in a tournament-style format. The proposed evaluation method achieves a high correlation with human preference, with a $\\rho$ score of 0.88. As an additional"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2406.15227","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-21T15:11:33Z","cross_cats_sorted":[],"title_canon_sha256":"c2d33e05dc31660f27379cb94dc33c607ad7e87461ab9cee4f92bb05f4b3db8a","abstract_canon_sha256":"770e40d116cc0af27bcbe9bfaa23c611ae76a0fd51b49558df3567f91f2b1134"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:30:17.572636Z","signature_b64":"GOwcFNWKwGClua6KF5oCci9Vn9yrctzoWTHfSGEn3yMjYNAlxGQLr9Vv+lV4ABLVq54Ql7hs+9sH+wU1p8kZAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"29c997ddfd48b7e82cd4b2da125534600ed3ad3f8305af194393f9356ac2bd41","last_reissued_at":"2026-07-05T09:30:17.572141Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:30:17.572141Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"A LLM-Based Ranking Method for the Evaluation of Automatic Counter-Narrative Generation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Aitor Soroa, Irune Zubiaga, Rodrigo Agerri","submitted_at":"2024-06-21T15:11:33Z","abstract_excerpt":"This paper proposes a novel approach to evaluate Counter Narrative (CN) generation using a Large Language Model (LLM) as an evaluator. We show that traditional automatic metrics correlate poorly with human judgements and fail to capture the nuanced relationship between generated CNs and human perception. To alleviate this, we introduce a model ranking pipeline based on pairwise comparisons of generated CNs from different models, organized in a tournament-style format. The proposed evaluation method achieves a high correlation with human preference, with a $\\rho$ score of 0.88. As an additional"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.15227","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.15227/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2406.15227","created_at":"2026-07-05T09:30:17.572200+00:00"},{"alias_kind":"arxiv_version","alias_value":"2406.15227v3","created_at":"2026-07-05T09:30:17.572200+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.15227","created_at":"2026-07-05T09:30:17.572200+00:00"},{"alias_kind":"pith_short_12","alias_value":"FHEZPXP5JC36","created_at":"2026-07-05T09:30:17.572200+00:00"},{"alias_kind":"pith_short_16","alias_value":"FHEZPXP5JC36QLGU","created_at":"2026-07-05T09:30:17.572200+00:00"},{"alias_kind":"pith_short_8","alias_value":"FHEZPXP5","created_at":"2026-07-05T09:30:17.572200+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2501.00697","citing_title":"PANDA -- Paired Anti-hate Narratives Dataset from Asia: Using an LLM-as-a-Judge to Create the First Chinese Counterspeech Dataset","ref_index":27,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/FHEZPXP5JC36QLGUWLNBEVJUMA","json":"https://pith.science/pith/FHEZPXP5JC36QLGUWLNBEVJUMA.json","graph_json":"https://pith.science/api/pith-number/FHEZPXP5JC36QLGUWLNBEVJUMA/graph.json","events_json":"https://pith.science/api/pith-number/FHEZPXP5JC36QLGUWLNBEVJUMA/events.json","paper":"https://pith.science/paper/FHEZPXP5"},"agent_actions":{"view_html":"https://pith.science/pith/FHEZPXP5JC36QLGUWLNBEVJUMA","download_json":"https://pith.science/pith/FHEZPXP5JC36QLGUWLNBEVJUMA.json","view_paper":"https://pith.science/paper/FHEZPXP5","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2406.15227&json=true","fetch_graph":"https://pith.science/api/pith-number/FHEZPXP5JC36QLGUWLNBEVJUMA/graph.json","fetch_events":"https://pith.science/api/pith-number/FHEZPXP5JC36QLGUWLNBEVJUMA/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/FHEZPXP5JC36QLGUWLNBEVJUMA/action/timestamp_anchor","attest_storage":"https://pith.science/pith/FHEZPXP5JC36QLGUWLNBEVJUMA/action/storage_attestation","attest_author":"https://pith.science/pith/FHEZPXP5JC36QLGUWLNBEVJUMA/action/author_attestation","sign_citation":"https://pith.science/pith/FHEZPXP5JC36QLGUWLNBEVJUMA/action/citation_signature","submit_replication":"https://pith.science/pith/FHEZPXP5JC36QLGUWLNBEVJUMA/action/replication_record"}},"created_at":"2026-07-05T09:30:17.572200+00:00","updated_at":"2026-07-05T09:30:17.572200+00:00"}