{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:4UJ4DGFII7LUPVCRIBRL55X3F6","short_pith_number":"pith:4UJ4DGFI","schema_version":"1.0","canonical_sha256":"e513c198a847d747d4514062bef6fb2f8763fe838132b21e1fb7ef9bd6fed424","source":{"kind":"arxiv","id":"2504.07825","version":1},"attestation_state":"computed","paper":{"title":"What the HellaSwag? On the Validity of Common-Sense Reasoning Benchmarks","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Ivan P. Yamshchikov, Mattia Nee, Pavel Chizhov, Pierre-Carl Langlais","submitted_at":"2025-04-10T15:01:46Z","abstract_excerpt":"Common-sense reasoning is a key language model capability because it encapsulates not just specific factual knowledge but rather general language and world understanding. Measuring common-sense reasoning, therefore, is crucial for language models of different sizes and applications. One of the most widely used benchmarks for evaluating such capabilities is HellaSwag; however, in this paper, we show that it has severe construct validity issues. These issues range from basic ungrammaticality and numerous typos to misleading prompts or equally correct options. Furthermore, we show that if models "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2504.07825","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-04-10T15:01:46Z","cross_cats_sorted":[],"title_canon_sha256":"95228dce548b1e43a5cca17fe1de4008a71ecb8631a421d3b0b1c2f3c5dd01a3","abstract_canon_sha256":"1a35559ac0ab74691d1804465c32cdbb38c95063356f8b96f325371238e4cf6f"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:47:15.432575Z","signature_b64":"i47F/PQYSo32Lh/TAKlp8SscgYAPpqYx4FbGexYDgn/v9Z/A2KLyx2Hc3Uj7/uaS12WzIGo64g4KTaU5IxvrBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"e513c198a847d747d4514062bef6fb2f8763fe838132b21e1fb7ef9bd6fed424","last_reissued_at":"2026-07-05T10:47:15.432084Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:47:15.432084Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"What the HellaSwag? On the Validity of Common-Sense Reasoning Benchmarks","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Ivan P. Yamshchikov, Mattia Nee, Pavel Chizhov, Pierre-Carl Langlais","submitted_at":"2025-04-10T15:01:46Z","abstract_excerpt":"Common-sense reasoning is a key language model capability because it encapsulates not just specific factual knowledge but rather general language and world understanding. Measuring common-sense reasoning, therefore, is crucial for language models of different sizes and applications. One of the most widely used benchmarks for evaluating such capabilities is HellaSwag; however, in this paper, we show that it has severe construct validity issues. These issues range from basic ungrammaticality and numerous typos to misleading prompts or equally correct options. Furthermore, we show that if models "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2504.07825","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2504.07825/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2504.07825","created_at":"2026-07-05T10:47:15.432145+00:00"},{"alias_kind":"arxiv_version","alias_value":"2504.07825v1","created_at":"2026-07-05T10:47:15.432145+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.07825","created_at":"2026-07-05T10:47:15.432145+00:00"},{"alias_kind":"pith_short_12","alias_value":"4UJ4DGFII7LU","created_at":"2026-07-05T10:47:15.432145+00:00"},{"alias_kind":"pith_short_16","alias_value":"4UJ4DGFII7LUPVCR","created_at":"2026-07-05T10:47:15.432145+00:00"},{"alias_kind":"pith_short_8","alias_value":"4UJ4DGFI","created_at":"2026-07-05T10:47:15.432145+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":2,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2605.27394","citing_title":"Human-AI Collaboration for Estimating Scientific Replicability","ref_index":15,"is_internal_anchor":true},{"citing_arxiv_id":"2606.11470","citing_title":"The Periodic Table of LLM Reasoning: A Structured Survey of Reasoning Paradigms, Methods, and Failure Modes","ref_index":36,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/4UJ4DGFII7LUPVCRIBRL55X3F6","json":"https://pith.science/pith/4UJ4DGFII7LUPVCRIBRL55X3F6.json","graph_json":"https://pith.science/api/pith-number/4UJ4DGFII7LUPVCRIBRL55X3F6/graph.json","events_json":"https://pith.science/api/pith-number/4UJ4DGFII7LUPVCRIBRL55X3F6/events.json","paper":"https://pith.science/paper/4UJ4DGFI"},"agent_actions":{"view_html":"https://pith.science/pith/4UJ4DGFII7LUPVCRIBRL55X3F6","download_json":"https://pith.science/pith/4UJ4DGFII7LUPVCRIBRL55X3F6.json","view_paper":"https://pith.science/paper/4UJ4DGFI","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2504.07825&json=true","fetch_graph":"https://pith.science/api/pith-number/4UJ4DGFII7LUPVCRIBRL55X3F6/graph.json","fetch_events":"https://pith.science/api/pith-number/4UJ4DGFII7LUPVCRIBRL55X3F6/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/4UJ4DGFII7LUPVCRIBRL55X3F6/action/timestamp_anchor","attest_storage":"https://pith.science/pith/4UJ4DGFII7LUPVCRIBRL55X3F6/action/storage_attestation","attest_author":"https://pith.science/pith/4UJ4DGFII7LUPVCRIBRL55X3F6/action/author_attestation","sign_citation":"https://pith.science/pith/4UJ4DGFII7LUPVCRIBRL55X3F6/action/citation_signature","submit_replication":"https://pith.science/pith/4UJ4DGFII7LUPVCRIBRL55X3F6/action/replication_record"}},"created_at":"2026-07-05T10:47:15.432145+00:00","updated_at":"2026-07-05T10:47:15.432145+00:00"}