{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:HTFKCUJI3FSNHQCO7CPATW5UGZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"144142726bdea7293ad01a5c1253d6988c58ce54480ad88df323dd11240799b9","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-21T15:58:37Z","title_canon_sha256":"240bee997740dde93d355e01f93f1a26da867d94a1adaafebbc6582d6c1e525b"},"schema_version":"1.0","source":{"id":"2402.13887","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.13887","created_at":"2026-07-05T08:41:43Z"},{"alias_kind":"arxiv_version","alias_value":"2402.13887v2","created_at":"2026-07-05T08:41:43Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.13887","created_at":"2026-07-05T08:41:43Z"},{"alias_kind":"pith_short_12","alias_value":"HTFKCUJI3FSN","created_at":"2026-07-05T08:41:43Z"},{"alias_kind":"pith_short_16","alias_value":"HTFKCUJI3FSNHQCO","created_at":"2026-07-05T08:41:43Z"},{"alias_kind":"pith_short_8","alias_value":"HTFKCUJI","created_at":"2026-07-05T08:41:43Z"}],"graph_snapshots":[{"event_id":"sha256:5e2b113a57a6da7c17b09c4b8ec2bc86a4753f32c8335bf1801df66f761bedb1","target":"graph","created_at":"2026-07-05T08:41:43Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.13887/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models (LLMs) have demonstrated remarkable capabilities across various applications, fundamentally reshaping the landscape of natural language processing (NLP) research. However, recent evaluation frameworks often rely on the output probabilities of LLMs for predictions, primarily due to computational constraints, diverging from real-world LLM usage scenarios. While widely employed, the efficacy of these probability-based evaluation strategies remains an open research question. This study aims to scrutinize the validity of such probability-based evaluation methods within the con","authors_text":"Alham Fikri Aji, Chenyang Lyu, Minghao Wu","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-21T15:58:37Z","title":"Beyond Probabilities: Unveiling the Misalignment in Evaluating Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.13887","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1d0796d72acf7109dd163be628ec68e9086c18b746abae14d9bd04cf9a4698e5","target":"record","created_at":"2026-07-05T08:41:43Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"144142726bdea7293ad01a5c1253d6988c58ce54480ad88df323dd11240799b9","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-21T15:58:37Z","title_canon_sha256":"240bee997740dde93d355e01f93f1a26da867d94a1adaafebbc6582d6c1e525b"},"schema_version":"1.0","source":{"id":"2402.13887","kind":"arxiv","version":2}},"canonical_sha256":"3ccaa15128d964d3c04ef89e09dbb436774b1f076632dd6e7ca1546b2cf3f0ff","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3ccaa15128d964d3c04ef89e09dbb436774b1f076632dd6e7ca1546b2cf3f0ff","first_computed_at":"2026-07-05T08:41:43.350253Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:41:43.350253Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"KTVduHE/MrJJ0J2QPKiJ8UJc7SsHQrJyAfUcMN2Gi5HWV0QuzkVu7AvrhPaO8XdrDPsKFfwyoLG8/AqeuUiIAw==","signature_status":"signed_v1","signed_at":"2026-07-05T08:41:43.350658Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.13887","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1d0796d72acf7109dd163be628ec68e9086c18b746abae14d9bd04cf9a4698e5","sha256:5e2b113a57a6da7c17b09c4b8ec2bc86a4753f32c8335bf1801df66f761bedb1"],"state_sha256":"42acec2d885e8650d9c4de4a519b84c1faf0e2e51715ef1eb3d9e0776c1f809d"}