{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:MC3A436NIHLFUX5HDKMESD3P4O","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c9acfa421f3dc5c7f4128ce566a237caf5fc067055caf1e4df1b9b8734cfb100","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-26T21:33:27Z","title_canon_sha256":"97288af3db381613560c19cfe0cf4be258e09bda89fe2700382cea1a78dab035"},"schema_version":"1.0","source":{"id":"2405.16702","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.16702","created_at":"2026-07-05T08:23:34Z"},{"alias_kind":"arxiv_version","alias_value":"2405.16702v1","created_at":"2026-07-05T08:23:34Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.16702","created_at":"2026-07-05T08:23:34Z"},{"alias_kind":"pith_short_12","alias_value":"MC3A436NIHLF","created_at":"2026-07-05T08:23:34Z"},{"alias_kind":"pith_short_16","alias_value":"MC3A436NIHLFUX5H","created_at":"2026-07-05T08:23:34Z"},{"alias_kind":"pith_short_8","alias_value":"MC3A436N","created_at":"2026-07-05T08:23:34Z"}],"graph_snapshots":[{"event_id":"sha256:077a02030de4d84e10dfae698cc1d0053ee8b8e97e024feb20022a36f12cd251","target":"graph","created_at":"2026-07-05T08:23:34Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.16702/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Open-domain question answering (Open-QA) is a common task for evaluating large language models (LLMs). However, current Open-QA evaluations are criticized for the ambiguity in questions and the lack of semantic understanding in evaluators. Complex evaluators, powered by foundation models or LLMs and pertaining to semantic equivalence, still deviate from human judgments by a large margin. We propose to study the entailment relations of answers to identify more informative and more general system answers, offering a much closer evaluation to human judgment on both NaturalQuestions and TriviaQA w","authors_text":"Denilson Barbosa, Peiran Yao","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-26T21:33:27Z","title":"Accurate and Nuanced Open-QA Evaluation Through Textual Entailment"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.16702","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:4ecdd0e6a7d82c163fc5aed83370bdf5ac6223c18f2c10f5fd1b7eda49b77f8a","target":"record","created_at":"2026-07-05T08:23:34Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c9acfa421f3dc5c7f4128ce566a237caf5fc067055caf1e4df1b9b8734cfb100","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-26T21:33:27Z","title_canon_sha256":"97288af3db381613560c19cfe0cf4be258e09bda89fe2700382cea1a78dab035"},"schema_version":"1.0","source":{"id":"2405.16702","kind":"arxiv","version":1}},"canonical_sha256":"60b60e6fcd41d65a5fa71a98490f6fe38e29bfa50e5a7c7d86739ee7beeb8e58","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"60b60e6fcd41d65a5fa71a98490f6fe38e29bfa50e5a7c7d86739ee7beeb8e58","first_computed_at":"2026-07-05T08:23:34.206537Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:23:34.206537Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"aozssDNNR5/3kSnSiRKfpmvlCKu0Nw2rcS5mB08jmbtZ9bPaDaqd6T6Wl/pkSVvEEG8of9G+G8w9t+pkISr7DQ==","signature_status":"signed_v1","signed_at":"2026-07-05T08:23:34.206945Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.16702","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:4ecdd0e6a7d82c163fc5aed83370bdf5ac6223c18f2c10f5fd1b7eda49b77f8a","sha256:077a02030de4d84e10dfae698cc1d0053ee8b8e97e024feb20022a36f12cd251"],"state_sha256":"86e9019bce149ca482a0e38c978d3da99155a9333f9d3a1b76f8860833940b14"}