{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:3B54YSARSKYSBYAAKQTM7HVJ6B","short_pith_number":"pith:3B54YSAR","schema_version":"1.0","canonical_sha256":"d87bcc481192b120e0005426cf9ea9f0599d37ddc9d7b479e445757250abb806","source":{"kind":"arxiv","id":"2506.22316","version":4},"attestation_state":"computed","paper":{"title":"Evaluating Scoring Bias in LLM-as-a-Judge","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Chao Chen, Haixiang Hu, Kailai Shao, Qingquan Li, Shaoyu Dou","submitted_at":"2025-06-27T15:25:23Z","abstract_excerpt":"The \"LLM-as-a-Judge\" paradigm, using Large Language Models (LLMs) as automated evaluators, is pivotal to LLM development, offering scalable feedback for complex tasks. However, the reliability of these judges is compromised by various biases. Existing research has heavily concentrated on biases in comparative evaluations. In contrast, scoring-based evaluations-which assign an absolute score and are often more practical in industrial applications-remain under-investigated. To address this gap, we undertake the first dedicated examination of scoring bias in LLM judges. We shift the focus from bi"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2506.22316","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-06-27T15:25:23Z","cross_cats_sorted":[],"title_canon_sha256":"eb7f2862f4197c3e4e334cdf15e7706e7e197e80013403fc1e407f784dfa8e4e","abstract_canon_sha256":"ec54702453ebdc278664a0cb29f0721e801cb2489811b05a0bd154ee523af03d"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-22T01:03:43.870577Z","signature_b64":"asQF+86iiWGs0c3rRhx5dyEA3BtvLHRwFj7feC5E+WLsL39siWPENvE9smp388HCva4n//TC4wh8gn5f6qbOBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d87bcc481192b120e0005426cf9ea9f0599d37ddc9d7b479e445757250abb806","last_reissued_at":"2026-05-22T01:03:43.869877Z","signature_status":"signed_v1","first_computed_at":"2026-05-22T01:03:43.869877Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Evaluating Scoring Bias in LLM-as-a-Judge","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Chao Chen, Haixiang Hu, Kailai Shao, Qingquan Li, Shaoyu Dou","submitted_at":"2025-06-27T15:25:23Z","abstract_excerpt":"The \"LLM-as-a-Judge\" paradigm, using Large Language Models (LLMs) as automated evaluators, is pivotal to LLM development, offering scalable feedback for complex tasks. However, the reliability of these judges is compromised by various biases. Existing research has heavily concentrated on biases in comparative evaluations. In contrast, scoring-based evaluations-which assign an absolute score and are often more practical in industrial applications-remain under-investigated. To address this gap, we undertake the first dedicated examination of scoring bias in LLM judges. We shift the focus from bi"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.22316","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.22316/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2506.22316","created_at":"2026-05-22T01:03:43.869985+00:00"},{"alias_kind":"arxiv_version","alias_value":"2506.22316v4","created_at":"2026-05-22T01:03:43.869985+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.22316","created_at":"2026-05-22T01:03:43.869985+00:00"},{"alias_kind":"pith_short_12","alias_value":"3B54YSARSKYS","created_at":"2026-05-22T01:03:43.869985+00:00"},{"alias_kind":"pith_short_16","alias_value":"3B54YSARSKYSBYAA","created_at":"2026-05-22T01:03:43.869985+00:00"},{"alias_kind":"pith_short_8","alias_value":"3B54YSAR","created_at":"2026-05-22T01:03:43.869985+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":7,"internal_anchor_count":7,"sample":[{"citing_arxiv_id":"2605.16386","citing_title":"Auditing Multimodal LLM Raters: Central Tendency Bias in Clinical Ordinal Scoring","ref_index":14,"is_internal_anchor":true},{"citing_arxiv_id":"2605.12667","citing_title":"ODRPO: Ordinal Decompositions of Discrete Rewards for Robust Policy Optimization","ref_index":8,"is_internal_anchor":true},{"citing_arxiv_id":"2604.13061","citing_title":"Token Statistics Reveal Conversational Drift in Multi-turn LLM Interaction","ref_index":15,"is_internal_anchor":true},{"citing_arxiv_id":"2605.12667","citing_title":"ODRPO: Ordinal Decompositions of Discrete Rewards for Robust Policy Optimization","ref_index":8,"is_internal_anchor":true},{"citing_arxiv_id":"2605.08522","citing_title":"Coordinates of Capability: A Unified MTMM-Geometric Framework for LLM Evaluation","ref_index":26,"is_internal_anchor":true},{"citing_arxiv_id":"2605.06161","citing_title":"Beyond Accuracy: Policy Invariance as a Reliability Test for LLM Safety Judges","ref_index":24,"is_internal_anchor":true},{"citing_arxiv_id":"2604.05593","citing_title":"Label Effects: Shared Heuristic Reliance in Trust Assessment by Humans and LLM-as-a-Judge","ref_index":26,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/3B54YSARSKYSBYAAKQTM7HVJ6B","json":"https://pith.science/pith/3B54YSARSKYSBYAAKQTM7HVJ6B.json","graph_json":"https://pith.science/api/pith-number/3B54YSARSKYSBYAAKQTM7HVJ6B/graph.json","events_json":"https://pith.science/api/pith-number/3B54YSARSKYSBYAAKQTM7HVJ6B/events.json","paper":"https://pith.science/paper/3B54YSAR"},"agent_actions":{"view_html":"https://pith.science/pith/3B54YSARSKYSBYAAKQTM7HVJ6B","download_json":"https://pith.science/pith/3B54YSARSKYSBYAAKQTM7HVJ6B.json","view_paper":"https://pith.science/paper/3B54YSAR","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2506.22316&json=true","fetch_graph":"https://pith.science/api/pith-number/3B54YSARSKYSBYAAKQTM7HVJ6B/graph.json","fetch_events":"https://pith.science/api/pith-number/3B54YSARSKYSBYAAKQTM7HVJ6B/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/3B54YSARSKYSBYAAKQTM7HVJ6B/action/timestamp_anchor","attest_storage":"https://pith.science/pith/3B54YSARSKYSBYAAKQTM7HVJ6B/action/storage_attestation","attest_author":"https://pith.science/pith/3B54YSARSKYSBYAAKQTM7HVJ6B/action/author_attestation","sign_citation":"https://pith.science/pith/3B54YSARSKYSBYAAKQTM7HVJ6B/action/citation_signature","submit_replication":"https://pith.science/pith/3B54YSARSKYSBYAAKQTM7HVJ6B/action/replication_record"}},"created_at":"2026-05-22T01:03:43.869985+00:00","updated_at":"2026-05-22T01:03:43.869985+00:00"}