{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:HCF4GHYCK7IGU3PTS42ZTEVQY3","short_pith_number":"pith:HCF4GHYC","canonical_record":{"source":{"id":"2410.15393","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-20T13:47:39Z","cross_cats_sorted":[],"title_canon_sha256":"a34512b51c76c6947a08a160ba8a517e1cb658442ffe91b9775199d191cdd08a","abstract_canon_sha256":"87afc614f8fd1ccef7cd3b4705f1a763a5c683a79f7f0d7bff8e75d1e29d3620"},"schema_version":"1.0"},"canonical_sha256":"388bc31f0257d06a6df397359992b0c6cbcf6bd2658e5c11589aa9fdedf40ab2","source":{"kind":"arxiv","id":"2410.15393","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.15393","created_at":"2026-07-05T09:23:18Z"},{"alias_kind":"arxiv_version","alias_value":"2410.15393v1","created_at":"2026-07-05T09:23:18Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.15393","created_at":"2026-07-05T09:23:18Z"},{"alias_kind":"pith_short_12","alias_value":"HCF4GHYCK7IG","created_at":"2026-07-05T09:23:18Z"},{"alias_kind":"pith_short_16","alias_value":"HCF4GHYCK7IGU3PT","created_at":"2026-07-05T09:23:18Z"},{"alias_kind":"pith_short_8","alias_value":"HCF4GHYC","created_at":"2026-07-05T09:23:18Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:HCF4GHYCK7IGU3PTS42ZTEVQY3","target":"record","payload":{"canonical_record":{"source":{"id":"2410.15393","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-20T13:47:39Z","cross_cats_sorted":[],"title_canon_sha256":"a34512b51c76c6947a08a160ba8a517e1cb658442ffe91b9775199d191cdd08a","abstract_canon_sha256":"87afc614f8fd1ccef7cd3b4705f1a763a5c683a79f7f0d7bff8e75d1e29d3620"},"schema_version":"1.0"},"canonical_sha256":"388bc31f0257d06a6df397359992b0c6cbcf6bd2658e5c11589aa9fdedf40ab2","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:23:18.856506Z","signature_b64":"OxWqEVC6K6lJPhWS+FztInb3FZmQvnIQ4W3KpugaLz4XBwNNw5f/Dy31xkPJdxuN9sCUhQcIxDlhY9UZNifiAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"388bc31f0257d06a6df397359992b0c6cbcf6bd2658e5c11589aa9fdedf40ab2","last_reissued_at":"2026-07-05T09:23:18.856070Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:23:18.856070Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.15393","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:23:18Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"pCNp1T9sxidzGMkm3u4s1Eo3LmpelN4S2iyp+nyYNgQ+wrI702I4jisS9i448b38nCFNvBKTZkot/5aS09LSAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T05:49:30.770303Z"},"content_sha256":"e36c00d9afc39adf0e2de8b855762a46ed3abec912f7a87bffd04c41d1d0bcd1","schema_version":"1.0","event_id":"sha256:e36c00d9afc39adf0e2de8b855762a46ed3abec912f7a87bffd04c41d1d0bcd1"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:HCF4GHYCK7IGU3PTS42ZTEVQY3","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"CalibraEval: Calibrating Prediction Distribution to Mitigate Selection Bias in LLMs-as-Judges","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Haitao Li, Junjie Chen, Qian Dong, Qingyao Ai, Yiqun Liu, Yujia Zhou, Zhumin Chu","submitted_at":"2024-10-20T13:47:39Z","abstract_excerpt":"The use of large language models (LLMs) as automated evaluation tools to assess the quality of generated natural language, known as LLMs-as-Judges, has demonstrated promising capabilities and is rapidly gaining widespread attention. However, when applied to pairwise comparisons of candidate responses, LLM-based evaluators often exhibit selection bias. Specifically, their judgments may become inconsistent when the option positions or ID tokens are swapped, compromising the effectiveness and fairness of the evaluation result. To address this challenge, we introduce CalibraEval, a novel label-fre"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.15393","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.15393/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:23:18Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"yQlEWMNLr/wTcVISQi7mI1jwt+dlJ2XGW6pS4cye2iyrQXrJOM+xcDw4Rp5uheoAVGjLoGHTaKLaWrpH3KlBBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T05:49:30.770963Z"},"content_sha256":"c59ab8866780f60e21f3fcbed453e18e398212206a587a1e821cad5dce086f11","schema_version":"1.0","event_id":"sha256:c59ab8866780f60e21f3fcbed453e18e398212206a587a1e821cad5dce086f11"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/HCF4GHYCK7IGU3PTS42ZTEVQY3/bundle.json","state_url":"https://pith.science/pith/HCF4GHYCK7IGU3PTS42ZTEVQY3/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/HCF4GHYCK7IGU3PTS42ZTEVQY3/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T05:49:30Z","links":{"resolver":"https://pith.science/pith/HCF4GHYCK7IGU3PTS42ZTEVQY3","bundle":"https://pith.science/pith/HCF4GHYCK7IGU3PTS42ZTEVQY3/bundle.json","state":"https://pith.science/pith/HCF4GHYCK7IGU3PTS42ZTEVQY3/state.json","well_known_bundle":"https://pith.science/.well-known/pith/HCF4GHYCK7IGU3PTS42ZTEVQY3/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:HCF4GHYCK7IGU3PTS42ZTEVQY3","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"87afc614f8fd1ccef7cd3b4705f1a763a5c683a79f7f0d7bff8e75d1e29d3620","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-20T13:47:39Z","title_canon_sha256":"a34512b51c76c6947a08a160ba8a517e1cb658442ffe91b9775199d191cdd08a"},"schema_version":"1.0","source":{"id":"2410.15393","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.15393","created_at":"2026-07-05T09:23:18Z"},{"alias_kind":"arxiv_version","alias_value":"2410.15393v1","created_at":"2026-07-05T09:23:18Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.15393","created_at":"2026-07-05T09:23:18Z"},{"alias_kind":"pith_short_12","alias_value":"HCF4GHYCK7IG","created_at":"2026-07-05T09:23:18Z"},{"alias_kind":"pith_short_16","alias_value":"HCF4GHYCK7IGU3PT","created_at":"2026-07-05T09:23:18Z"},{"alias_kind":"pith_short_8","alias_value":"HCF4GHYC","created_at":"2026-07-05T09:23:18Z"}],"graph_snapshots":[{"event_id":"sha256:c59ab8866780f60e21f3fcbed453e18e398212206a587a1e821cad5dce086f11","target":"graph","created_at":"2026-07-05T09:23:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.15393/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The use of large language models (LLMs) as automated evaluation tools to assess the quality of generated natural language, known as LLMs-as-Judges, has demonstrated promising capabilities and is rapidly gaining widespread attention. However, when applied to pairwise comparisons of candidate responses, LLM-based evaluators often exhibit selection bias. Specifically, their judgments may become inconsistent when the option positions or ID tokens are swapped, compromising the effectiveness and fairness of the evaluation result. To address this challenge, we introduce CalibraEval, a novel label-fre","authors_text":"Haitao Li, Junjie Chen, Qian Dong, Qingyao Ai, Yiqun Liu, Yujia Zhou, Zhumin Chu","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-20T13:47:39Z","title":"CalibraEval: Calibrating Prediction Distribution to Mitigate Selection Bias in LLMs-as-Judges"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.15393","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:e36c00d9afc39adf0e2de8b855762a46ed3abec912f7a87bffd04c41d1d0bcd1","target":"record","created_at":"2026-07-05T09:23:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"87afc614f8fd1ccef7cd3b4705f1a763a5c683a79f7f0d7bff8e75d1e29d3620","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-20T13:47:39Z","title_canon_sha256":"a34512b51c76c6947a08a160ba8a517e1cb658442ffe91b9775199d191cdd08a"},"schema_version":"1.0","source":{"id":"2410.15393","kind":"arxiv","version":1}},"canonical_sha256":"388bc31f0257d06a6df397359992b0c6cbcf6bd2658e5c11589aa9fdedf40ab2","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"388bc31f0257d06a6df397359992b0c6cbcf6bd2658e5c11589aa9fdedf40ab2","first_computed_at":"2026-07-05T09:23:18.856070Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:23:18.856070Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"OxWqEVC6K6lJPhWS+FztInb3FZmQvnIQ4W3KpugaLz4XBwNNw5f/Dy31xkPJdxuN9sCUhQcIxDlhY9UZNifiAA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:23:18.856506Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.15393","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:e36c00d9afc39adf0e2de8b855762a46ed3abec912f7a87bffd04c41d1d0bcd1","sha256:c59ab8866780f60e21f3fcbed453e18e398212206a587a1e821cad5dce086f11"],"state_sha256":"2b20ef121b42e41130fabba8d43df7cf65e97bb99da3b504b7ab6bf35a462443"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Dwy7neHjhBnZgJ22JlPjPqiqandKHMEUB8IlgFx8ATGeQ/f0xRzWhTp5BBH+qAwFJ0zk7eOE63usm2Vt+jgdCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T05:49:30.774094Z","bundle_sha256":"1393a311351d744275abb280c2847c0237ddea772560ccdbfc45f886a173114c"}}