{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:KKSNWKSO6VPSK4SVVDKVMATOYZ","short_pith_number":"pith:KKSNWKSO","canonical_record":{"source":{"id":"2502.10709","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-15T07:45:20Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"411008f1c72b7d4e7a898177cd2346bd384d311224753e001d67c90203c1a480","abstract_canon_sha256":"07826b351a5d0f1a162f86c1314f4fe3d687c384ebd3cef0510f53338a1f3971"},"schema_version":"1.0"},"canonical_sha256":"52a4db2a4ef55f257255a8d556026ec6668b3fd790cadced44c2f5d0a72786ee","source":{"kind":"arxiv","id":"2502.10709","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.10709","created_at":"2026-07-05T10:22:13Z"},{"alias_kind":"arxiv_version","alias_value":"2502.10709v2","created_at":"2026-07-05T10:22:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.10709","created_at":"2026-07-05T10:22:13Z"},{"alias_kind":"pith_short_12","alias_value":"KKSNWKSO6VPS","created_at":"2026-07-05T10:22:13Z"},{"alias_kind":"pith_short_16","alias_value":"KKSNWKSO6VPSK4SV","created_at":"2026-07-05T10:22:13Z"},{"alias_kind":"pith_short_8","alias_value":"KKSNWKSO","created_at":"2026-07-05T10:22:13Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:KKSNWKSO6VPSK4SVVDKVMATOYZ","target":"record","payload":{"canonical_record":{"source":{"id":"2502.10709","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-15T07:45:20Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"411008f1c72b7d4e7a898177cd2346bd384d311224753e001d67c90203c1a480","abstract_canon_sha256":"07826b351a5d0f1a162f86c1314f4fe3d687c384ebd3cef0510f53338a1f3971"},"schema_version":"1.0"},"canonical_sha256":"52a4db2a4ef55f257255a8d556026ec6668b3fd790cadced44c2f5d0a72786ee","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:22:13.502784Z","signature_b64":"1g5umrRugiaIezRupdCV146NEORX7JKEkeyBiqP5Dnj/QZWtxbfM7iNrQcFZnikZ3bkx05eqC+53mxT5cYxTAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"52a4db2a4ef55f257255a8d556026ec6668b3fd790cadced44c2f5d0a72786ee","last_reissued_at":"2026-07-05T10:22:13.502164Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:22:13.502164Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2502.10709","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:22:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"zzGm7u4z7E2ZPnxhtGjrQDYCpq4vsXWCUOHfWjvqB4WQhUedjQccIPG6ucshAR2cKcv5Mo1qzV7yXqcV14AiBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T00:22:19.204199Z"},"content_sha256":"83e2e01e453042f92413514c359d86b611eacde91e4fde0b45c737d5d0254018","schema_version":"1.0","event_id":"sha256:83e2e01e453042f92413514c359d86b611eacde91e4fde0b45c737d5d0254018"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:KKSNWKSO6VPSK4SVVDKVMATOYZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"An Empirical Analysis of Uncertainty in Large Language Model Evaluations","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Linyi Yang, Qingqiu Li, Qiujie Xie, Yuejie Zhang, Yue Zhang, Zhuohao Yu","submitted_at":"2025-02-15T07:45:20Z","abstract_excerpt":"As LLM-as-a-Judge emerges as a new paradigm for assessing large language models (LLMs), concerns have been raised regarding the alignment, bias, and stability of LLM evaluators. While substantial work has focused on alignment and bias, little research has concentrated on the stability of LLM evaluators. In this paper, we conduct extensive experiments involving 9 widely used LLM evaluators across 2 different evaluation settings to investigate the uncertainty in model-based LLM evaluations. We pinpoint that LLM evaluators exhibit varying uncertainty based on model families and sizes. With carefu"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.10709","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.10709/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:22:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jqftEAGE478TFilJ7DLQdpdGtAbU77Y3ji/DSxo/beHlUWtzMAIppN/Fl/IYZKiB8M4CpikzuERz0DLjwFviCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T00:22:19.204725Z"},"content_sha256":"a38a2899813b7c3fa31c8b3f794f061aed00bc1d6c76ef5eac375f1e52d64fba","schema_version":"1.0","event_id":"sha256:a38a2899813b7c3fa31c8b3f794f061aed00bc1d6c76ef5eac375f1e52d64fba"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/KKSNWKSO6VPSK4SVVDKVMATOYZ/bundle.json","state_url":"https://pith.science/pith/KKSNWKSO6VPSK4SVVDKVMATOYZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/KKSNWKSO6VPSK4SVVDKVMATOYZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-10T00:22:19Z","links":{"resolver":"https://pith.science/pith/KKSNWKSO6VPSK4SVVDKVMATOYZ","bundle":"https://pith.science/pith/KKSNWKSO6VPSK4SVVDKVMATOYZ/bundle.json","state":"https://pith.science/pith/KKSNWKSO6VPSK4SVVDKVMATOYZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/KKSNWKSO6VPSK4SVVDKVMATOYZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:KKSNWKSO6VPSK4SVVDKVMATOYZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"07826b351a5d0f1a162f86c1314f4fe3d687c384ebd3cef0510f53338a1f3971","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-15T07:45:20Z","title_canon_sha256":"411008f1c72b7d4e7a898177cd2346bd384d311224753e001d67c90203c1a480"},"schema_version":"1.0","source":{"id":"2502.10709","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.10709","created_at":"2026-07-05T10:22:13Z"},{"alias_kind":"arxiv_version","alias_value":"2502.10709v2","created_at":"2026-07-05T10:22:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.10709","created_at":"2026-07-05T10:22:13Z"},{"alias_kind":"pith_short_12","alias_value":"KKSNWKSO6VPS","created_at":"2026-07-05T10:22:13Z"},{"alias_kind":"pith_short_16","alias_value":"KKSNWKSO6VPSK4SV","created_at":"2026-07-05T10:22:13Z"},{"alias_kind":"pith_short_8","alias_value":"KKSNWKSO","created_at":"2026-07-05T10:22:13Z"}],"graph_snapshots":[{"event_id":"sha256:a38a2899813b7c3fa31c8b3f794f061aed00bc1d6c76ef5eac375f1e52d64fba","target":"graph","created_at":"2026-07-05T10:22:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2502.10709/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"As LLM-as-a-Judge emerges as a new paradigm for assessing large language models (LLMs), concerns have been raised regarding the alignment, bias, and stability of LLM evaluators. While substantial work has focused on alignment and bias, little research has concentrated on the stability of LLM evaluators. In this paper, we conduct extensive experiments involving 9 widely used LLM evaluators across 2 different evaluation settings to investigate the uncertainty in model-based LLM evaluations. We pinpoint that LLM evaluators exhibit varying uncertainty based on model families and sizes. With carefu","authors_text":"Linyi Yang, Qingqiu Li, Qiujie Xie, Yuejie Zhang, Yue Zhang, Zhuohao Yu","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-15T07:45:20Z","title":"An Empirical Analysis of Uncertainty in Large Language Model Evaluations"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.10709","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:83e2e01e453042f92413514c359d86b611eacde91e4fde0b45c737d5d0254018","target":"record","created_at":"2026-07-05T10:22:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"07826b351a5d0f1a162f86c1314f4fe3d687c384ebd3cef0510f53338a1f3971","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-15T07:45:20Z","title_canon_sha256":"411008f1c72b7d4e7a898177cd2346bd384d311224753e001d67c90203c1a480"},"schema_version":"1.0","source":{"id":"2502.10709","kind":"arxiv","version":2}},"canonical_sha256":"52a4db2a4ef55f257255a8d556026ec6668b3fd790cadced44c2f5d0a72786ee","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"52a4db2a4ef55f257255a8d556026ec6668b3fd790cadced44c2f5d0a72786ee","first_computed_at":"2026-07-05T10:22:13.502164Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:22:13.502164Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"1g5umrRugiaIezRupdCV146NEORX7JKEkeyBiqP5Dnj/QZWtxbfM7iNrQcFZnikZ3bkx05eqC+53mxT5cYxTAg==","signature_status":"signed_v1","signed_at":"2026-07-05T10:22:13.502784Z","signed_message":"canonical_sha256_bytes"},"source_id":"2502.10709","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:83e2e01e453042f92413514c359d86b611eacde91e4fde0b45c737d5d0254018","sha256:a38a2899813b7c3fa31c8b3f794f061aed00bc1d6c76ef5eac375f1e52d64fba"],"state_sha256":"b68dd3b5b079bc62d7b95716dc378c523127492086a990a3a3d7f6757275a345"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"2vsU7OcT/i2vYQcx6ea3B22GPBrWqgVATqvO3WECdVJRqFXvgmFK9tSOLGvtaY61YjTuse0t7W2lJ0siR8pSCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-10T00:22:19.208871Z","bundle_sha256":"e47b9eb8d7b115d6eb89df24f84d0a8c505abd1ad53986746673059935d8a4a8"}}