{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:HTFKCUJI3FSNHQCO7CPATW5UGZ","short_pith_number":"pith:HTFKCUJI","canonical_record":{"source":{"id":"2402.13887","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-21T15:58:37Z","cross_cats_sorted":[],"title_canon_sha256":"240bee997740dde93d355e01f93f1a26da867d94a1adaafebbc6582d6c1e525b","abstract_canon_sha256":"144142726bdea7293ad01a5c1253d6988c58ce54480ad88df323dd11240799b9"},"schema_version":"1.0"},"canonical_sha256":"3ccaa15128d964d3c04ef89e09dbb436774b1f076632dd6e7ca1546b2cf3f0ff","source":{"kind":"arxiv","id":"2402.13887","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.13887","created_at":"2026-07-05T08:41:43Z"},{"alias_kind":"arxiv_version","alias_value":"2402.13887v2","created_at":"2026-07-05T08:41:43Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.13887","created_at":"2026-07-05T08:41:43Z"},{"alias_kind":"pith_short_12","alias_value":"HTFKCUJI3FSN","created_at":"2026-07-05T08:41:43Z"},{"alias_kind":"pith_short_16","alias_value":"HTFKCUJI3FSNHQCO","created_at":"2026-07-05T08:41:43Z"},{"alias_kind":"pith_short_8","alias_value":"HTFKCUJI","created_at":"2026-07-05T08:41:43Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:HTFKCUJI3FSNHQCO7CPATW5UGZ","target":"record","payload":{"canonical_record":{"source":{"id":"2402.13887","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-21T15:58:37Z","cross_cats_sorted":[],"title_canon_sha256":"240bee997740dde93d355e01f93f1a26da867d94a1adaafebbc6582d6c1e525b","abstract_canon_sha256":"144142726bdea7293ad01a5c1253d6988c58ce54480ad88df323dd11240799b9"},"schema_version":"1.0"},"canonical_sha256":"3ccaa15128d964d3c04ef89e09dbb436774b1f076632dd6e7ca1546b2cf3f0ff","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:41:43.350658Z","signature_b64":"KTVduHE/MrJJ0J2QPKiJ8UJc7SsHQrJyAfUcMN2Gi5HWV0QuzkVu7AvrhPaO8XdrDPsKFfwyoLG8/AqeuUiIAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3ccaa15128d964d3c04ef89e09dbb436774b1f076632dd6e7ca1546b2cf3f0ff","last_reissued_at":"2026-07-05T08:41:43.350253Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:41:43.350253Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2402.13887","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:41:43Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"RWiYGqyaesbMmfPaxP4ZkpYpezMzKXKgvt1mY5ZZBXjaT83VxpTD+zZIHLLB3wEeKCnCvEXW4bMifw2pvg0+AA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T00:37:15.232012Z"},"content_sha256":"1d0796d72acf7109dd163be628ec68e9086c18b746abae14d9bd04cf9a4698e5","schema_version":"1.0","event_id":"sha256:1d0796d72acf7109dd163be628ec68e9086c18b746abae14d9bd04cf9a4698e5"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:HTFKCUJI3FSNHQCO7CPATW5UGZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Beyond Probabilities: Unveiling the Misalignment in Evaluating Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Alham Fikri Aji, Chenyang Lyu, Minghao Wu","submitted_at":"2024-02-21T15:58:37Z","abstract_excerpt":"Large Language Models (LLMs) have demonstrated remarkable capabilities across various applications, fundamentally reshaping the landscape of natural language processing (NLP) research. However, recent evaluation frameworks often rely on the output probabilities of LLMs for predictions, primarily due to computational constraints, diverging from real-world LLM usage scenarios. While widely employed, the efficacy of these probability-based evaluation strategies remains an open research question. This study aims to scrutinize the validity of such probability-based evaluation methods within the con"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.13887","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.13887/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:41:43Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"itAvTB1bLuLb3c86uXTPX/HBEPnUAddtJiskr9cS3JGJv+2KsbLFJ3S5YiX2WwLTScW7df8YUXFPpLnwz000DQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T00:37:15.232599Z"},"content_sha256":"5e2b113a57a6da7c17b09c4b8ec2bc86a4753f32c8335bf1801df66f761bedb1","schema_version":"1.0","event_id":"sha256:5e2b113a57a6da7c17b09c4b8ec2bc86a4753f32c8335bf1801df66f761bedb1"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/HTFKCUJI3FSNHQCO7CPATW5UGZ/bundle.json","state_url":"https://pith.science/pith/HTFKCUJI3FSNHQCO7CPATW5UGZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/HTFKCUJI3FSNHQCO7CPATW5UGZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T00:37:15Z","links":{"resolver":"https://pith.science/pith/HTFKCUJI3FSNHQCO7CPATW5UGZ","bundle":"https://pith.science/pith/HTFKCUJI3FSNHQCO7CPATW5UGZ/bundle.json","state":"https://pith.science/pith/HTFKCUJI3FSNHQCO7CPATW5UGZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/HTFKCUJI3FSNHQCO7CPATW5UGZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:HTFKCUJI3FSNHQCO7CPATW5UGZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"144142726bdea7293ad01a5c1253d6988c58ce54480ad88df323dd11240799b9","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-21T15:58:37Z","title_canon_sha256":"240bee997740dde93d355e01f93f1a26da867d94a1adaafebbc6582d6c1e525b"},"schema_version":"1.0","source":{"id":"2402.13887","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.13887","created_at":"2026-07-05T08:41:43Z"},{"alias_kind":"arxiv_version","alias_value":"2402.13887v2","created_at":"2026-07-05T08:41:43Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.13887","created_at":"2026-07-05T08:41:43Z"},{"alias_kind":"pith_short_12","alias_value":"HTFKCUJI3FSN","created_at":"2026-07-05T08:41:43Z"},{"alias_kind":"pith_short_16","alias_value":"HTFKCUJI3FSNHQCO","created_at":"2026-07-05T08:41:43Z"},{"alias_kind":"pith_short_8","alias_value":"HTFKCUJI","created_at":"2026-07-05T08:41:43Z"}],"graph_snapshots":[{"event_id":"sha256:5e2b113a57a6da7c17b09c4b8ec2bc86a4753f32c8335bf1801df66f761bedb1","target":"graph","created_at":"2026-07-05T08:41:43Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.13887/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models (LLMs) have demonstrated remarkable capabilities across various applications, fundamentally reshaping the landscape of natural language processing (NLP) research. However, recent evaluation frameworks often rely on the output probabilities of LLMs for predictions, primarily due to computational constraints, diverging from real-world LLM usage scenarios. While widely employed, the efficacy of these probability-based evaluation strategies remains an open research question. This study aims to scrutinize the validity of such probability-based evaluation methods within the con","authors_text":"Alham Fikri Aji, Chenyang Lyu, Minghao Wu","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-21T15:58:37Z","title":"Beyond Probabilities: Unveiling the Misalignment in Evaluating Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.13887","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1d0796d72acf7109dd163be628ec68e9086c18b746abae14d9bd04cf9a4698e5","target":"record","created_at":"2026-07-05T08:41:43Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"144142726bdea7293ad01a5c1253d6988c58ce54480ad88df323dd11240799b9","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-21T15:58:37Z","title_canon_sha256":"240bee997740dde93d355e01f93f1a26da867d94a1adaafebbc6582d6c1e525b"},"schema_version":"1.0","source":{"id":"2402.13887","kind":"arxiv","version":2}},"canonical_sha256":"3ccaa15128d964d3c04ef89e09dbb436774b1f076632dd6e7ca1546b2cf3f0ff","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3ccaa15128d964d3c04ef89e09dbb436774b1f076632dd6e7ca1546b2cf3f0ff","first_computed_at":"2026-07-05T08:41:43.350253Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:41:43.350253Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"KTVduHE/MrJJ0J2QPKiJ8UJc7SsHQrJyAfUcMN2Gi5HWV0QuzkVu7AvrhPaO8XdrDPsKFfwyoLG8/AqeuUiIAw==","signature_status":"signed_v1","signed_at":"2026-07-05T08:41:43.350658Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.13887","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1d0796d72acf7109dd163be628ec68e9086c18b746abae14d9bd04cf9a4698e5","sha256:5e2b113a57a6da7c17b09c4b8ec2bc86a4753f32c8335bf1801df66f761bedb1"],"state_sha256":"42acec2d885e8650d9c4de4a519b84c1faf0e2e51715ef1eb3d9e0776c1f809d"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4QR8H/UN0tfyqrepS292R6v1WZb9wiTm6+PJG+SUO5K7j0yVS1YXxDERB8Lt1SX3wOCgbyzAqjX9dqizoTdTDA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T00:37:15.238096Z","bundle_sha256":"55f689a034c9a35325bf585def393d5e97b287c33ade9cf5074dfbdb4aad16ee"}}