{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:46ZMP7C2D65XKGPK2TOIBCXBS5","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"939fb41afea11a8163fd64ed78b1e415a31400df34e1a02f573ac7b22bd509a0","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-30T13:57:28Z","title_canon_sha256":"c1d16564d77ec48dadedaae6b684d7632ef8bf88bb57a1e818bc6e2f1024f67f"},"schema_version":"1.0","source":{"id":"2506.23864","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.23864","created_at":"2026-07-05T11:29:29Z"},{"alias_kind":"arxiv_version","alias_value":"2506.23864v1","created_at":"2026-07-05T11:29:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.23864","created_at":"2026-07-05T11:29:29Z"},{"alias_kind":"pith_short_12","alias_value":"46ZMP7C2D65X","created_at":"2026-07-05T11:29:29Z"},{"alias_kind":"pith_short_16","alias_value":"46ZMP7C2D65XKGPK","created_at":"2026-07-05T11:29:29Z"},{"alias_kind":"pith_short_8","alias_value":"46ZMP7C2","created_at":"2026-07-05T11:29:29Z"}],"graph_snapshots":[{"event_id":"sha256:4edaef5a3bd2c2b4781e0b77414b06c8b34fae8248d8802f04e7524672e9229a","target":"graph","created_at":"2026-07-05T11:29:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2506.23864/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We conduct a systematic audit of three widely used reasoning benchmarks, SocialIQa, FauxPas-EAI, and ToMi, and uncover pervasive flaws in both benchmark items and evaluation methodology. Using five LLMs (GPT-{3, 3.5, 4, o1}, and LLaMA 3.1) as diagnostic tools, we identify structural, semantic, and pragmatic issues in benchmark design (e.g., duplicated items, ambiguous wording, and implausible answers), as well as scoring procedures that prioritize output form over reasoning process. Through systematic human annotation and re-evaluation on cleaned benchmark subsets, we find that model scores of","authors_text":"Edoardo Cecchinato, Giuseppe Riccardi, Lucia Hornikova, Seyed Mahed Mousavi","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-30T13:57:28Z","title":"Garbage In, Reasoning Out? Why Benchmark Scores are Unreliable and What to Do About It"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.23864","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:0300385f93afde1268aa5500af572cbd7259811fbd2723b42796ce58138ba938","target":"record","created_at":"2026-07-05T11:29:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"939fb41afea11a8163fd64ed78b1e415a31400df34e1a02f573ac7b22bd509a0","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-30T13:57:28Z","title_canon_sha256":"c1d16564d77ec48dadedaae6b684d7632ef8bf88bb57a1e818bc6e2f1024f67f"},"schema_version":"1.0","source":{"id":"2506.23864","kind":"arxiv","version":1}},"canonical_sha256":"e7b2c7fc5a1fbb7519ead4dc808ae1976fad03044f70daf2884f8392330c093b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"e7b2c7fc5a1fbb7519ead4dc808ae1976fad03044f70daf2884f8392330c093b","first_computed_at":"2026-07-05T11:29:29.246752Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:29:29.246752Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"9Hjzfws5wp6qfDqFffS2hicfPRwk5oNhX6QmMQyQ4PpYxFUCHD1QMjxzMvtwwRLMh1F4D1Ih/zlIpYhIj3k0BQ==","signature_status":"signed_v1","signed_at":"2026-07-05T11:29:29.247285Z","signed_message":"canonical_sha256_bytes"},"source_id":"2506.23864","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:0300385f93afde1268aa5500af572cbd7259811fbd2723b42796ce58138ba938","sha256:4edaef5a3bd2c2b4781e0b77414b06c8b34fae8248d8802f04e7524672e9229a"],"state_sha256":"92d8714f8655d39efb644c30dd595487801a2b37ef2ccb3ef532d4fa47d518a2"}