{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2021:J77RV6BVJXFMBCPIKQ4E6JN32X","short_pith_number":"pith:J77RV6BV","schema_version":"1.0","canonical_sha256":"4fff1af8354dcac089e854384f25bbd5c21059d05248ba338c9f03bfe6a47f95","source":{"kind":"arxiv","id":"2108.06130","version":3},"attestation_state":"computed","paper":{"title":"Semantic Answer Similarity for Evaluating Question Answering Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.IR"],"primary_cat":"cs.CL","authors_text":"Julian Gutsch, Julian Risch, Malte Pietsch, Timo M\\\"oller","submitted_at":"2021-08-13T09:12:27Z","abstract_excerpt":"The evaluation of question answering models compares ground-truth annotations with model predictions. However, as of today, this comparison is mostly lexical-based and therefore misses out on answers that have no lexical overlap but are still semantically similar, thus treating correct answers as false. This underestimation of the true performance of models hinders user acceptance in applications and complicates a fair comparison of different models. Therefore, there is a need for an evaluation metric that is based on semantics instead of pure string similarity. In this short paper, we present"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2108.06130","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-08-13T09:12:27Z","cross_cats_sorted":["cs.IR"],"title_canon_sha256":"9176fb02d5d12ce0a75c15e0831de81af75210692f3fe2bddd83aea7840797ae","abstract_canon_sha256":"ffa4f27db2ab9587755f9f71348c4363e289c036f688c327fd7ffd9b94e2b87c"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T03:24:31.078212Z","signature_b64":"fT5KxeSfRhi01cIukc2hri332KEhG08xL85Fa+kOeYDy0BaTWnMdL9Gyi3Hp6M2jqN1yKKd27qB5zgUMkbXrCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4fff1af8354dcac089e854384f25bbd5c21059d05248ba338c9f03bfe6a47f95","last_reissued_at":"2026-07-05T03:24:31.077696Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T03:24:31.077696Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Semantic Answer Similarity for Evaluating Question Answering Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.IR"],"primary_cat":"cs.CL","authors_text":"Julian Gutsch, Julian Risch, Malte Pietsch, Timo M\\\"oller","submitted_at":"2021-08-13T09:12:27Z","abstract_excerpt":"The evaluation of question answering models compares ground-truth annotations with model predictions. However, as of today, this comparison is mostly lexical-based and therefore misses out on answers that have no lexical overlap but are still semantically similar, thus treating correct answers as false. This underestimation of the true performance of models hinders user acceptance in applications and complicates a fair comparison of different models. Therefore, there is a need for an evaluation metric that is based on semantics instead of pure string similarity. In this short paper, we present"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2108.06130","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2108.06130/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2108.06130","created_at":"2026-07-05T03:24:31.077770+00:00"},{"alias_kind":"arxiv_version","alias_value":"2108.06130v3","created_at":"2026-07-05T03:24:31.077770+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2108.06130","created_at":"2026-07-05T03:24:31.077770+00:00"},{"alias_kind":"pith_short_12","alias_value":"J77RV6BVJXFM","created_at":"2026-07-05T03:24:31.077770+00:00"},{"alias_kind":"pith_short_16","alias_value":"J77RV6BVJXFMBCPI","created_at":"2026-07-05T03:24:31.077770+00:00"},{"alias_kind":"pith_short_8","alias_value":"J77RV6BV","created_at":"2026-07-05T03:24:31.077770+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/J77RV6BVJXFMBCPIKQ4E6JN32X","json":"https://pith.science/pith/J77RV6BVJXFMBCPIKQ4E6JN32X.json","graph_json":"https://pith.science/api/pith-number/J77RV6BVJXFMBCPIKQ4E6JN32X/graph.json","events_json":"https://pith.science/api/pith-number/J77RV6BVJXFMBCPIKQ4E6JN32X/events.json","paper":"https://pith.science/paper/J77RV6BV"},"agent_actions":{"view_html":"https://pith.science/pith/J77RV6BVJXFMBCPIKQ4E6JN32X","download_json":"https://pith.science/pith/J77RV6BVJXFMBCPIKQ4E6JN32X.json","view_paper":"https://pith.science/paper/J77RV6BV","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2108.06130&json=true","fetch_graph":"https://pith.science/api/pith-number/J77RV6BVJXFMBCPIKQ4E6JN32X/graph.json","fetch_events":"https://pith.science/api/pith-number/J77RV6BVJXFMBCPIKQ4E6JN32X/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/J77RV6BVJXFMBCPIKQ4E6JN32X/action/timestamp_anchor","attest_storage":"https://pith.science/pith/J77RV6BVJXFMBCPIKQ4E6JN32X/action/storage_attestation","attest_author":"https://pith.science/pith/J77RV6BVJXFMBCPIKQ4E6JN32X/action/author_attestation","sign_citation":"https://pith.science/pith/J77RV6BVJXFMBCPIKQ4E6JN32X/action/citation_signature","submit_replication":"https://pith.science/pith/J77RV6BVJXFMBCPIKQ4E6JN32X/action/replication_record"}},"created_at":"2026-07-05T03:24:31.077770+00:00","updated_at":"2026-07-05T03:24:31.077770+00:00"}