{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:C7CFWE2BWWNSU43L5NQAS3ZCWO","short_pith_number":"pith:C7CFWE2B","schema_version":"1.0","canonical_sha256":"17c45b1341b59b2a736beb60096f22b3a6f9b1df38fb6e6b30d2a6cf1390bc03","source":{"kind":"arxiv","id":"2412.00543","version":1},"attestation_state":"computed","paper":{"title":"Evaluating the Consistency of LLM Evaluators","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"James Thorne, Jiwoo Hong, Noah Lee","submitted_at":"2024-11-30T17:29:08Z","abstract_excerpt":"Large language models (LLMs) have shown potential as general evaluators along with the evident benefits of speed and cost. While their correlation against human annotators has been widely studied, consistency as evaluators is still understudied, raising concerns about the reliability of LLM evaluators. In this paper, we conduct extensive studies on the two aspects of consistency in LLM evaluations, Self-Consistency (SC) and Inter-scale Consistency (IC), on different scoring scales and criterion granularity with open-source and proprietary models. Our comprehensive analysis demonstrates that st"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2412.00543","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-30T17:29:08Z","cross_cats_sorted":[],"title_canon_sha256":"202f60efdc0c4b9bb1e7be5e74712a18be99a10c01b3c77005392b8bd8cef2f6","abstract_canon_sha256":"3e632266776fedee082d5213c7c392f0152f26e201de90c22ac64a7453088ed8"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:42:43.409430Z","signature_b64":"BgI4vAXQaC2h6X34aHC2AvH7hXBbcyb2XnwFfxXNehHKrXaHJ1PR9JGvkYslYjps8CpnWJHi5Ls8d6+cp+IGBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"17c45b1341b59b2a736beb60096f22b3a6f9b1df38fb6e6b30d2a6cf1390bc03","last_reissued_at":"2026-07-05T09:42:43.408925Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:42:43.408925Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Evaluating the Consistency of LLM Evaluators","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"James Thorne, Jiwoo Hong, Noah Lee","submitted_at":"2024-11-30T17:29:08Z","abstract_excerpt":"Large language models (LLMs) have shown potential as general evaluators along with the evident benefits of speed and cost. While their correlation against human annotators has been widely studied, consistency as evaluators is still understudied, raising concerns about the reliability of LLM evaluators. In this paper, we conduct extensive studies on the two aspects of consistency in LLM evaluations, Self-Consistency (SC) and Inter-scale Consistency (IC), on different scoring scales and criterion granularity with open-source and proprietary models. Our comprehensive analysis demonstrates that st"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.00543","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2412.00543/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2412.00543","created_at":"2026-07-05T09:42:43.408983+00:00"},{"alias_kind":"arxiv_version","alias_value":"2412.00543v1","created_at":"2026-07-05T09:42:43.408983+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.00543","created_at":"2026-07-05T09:42:43.408983+00:00"},{"alias_kind":"pith_short_12","alias_value":"C7CFWE2BWWNS","created_at":"2026-07-05T09:42:43.408983+00:00"},{"alias_kind":"pith_short_16","alias_value":"C7CFWE2BWWNSU43L","created_at":"2026-07-05T09:42:43.408983+00:00"},{"alias_kind":"pith_short_8","alias_value":"C7CFWE2B","created_at":"2026-07-05T09:42:43.408983+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2509.09705","citing_title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","ref_index":18,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/C7CFWE2BWWNSU43L5NQAS3ZCWO","json":"https://pith.science/pith/C7CFWE2BWWNSU43L5NQAS3ZCWO.json","graph_json":"https://pith.science/api/pith-number/C7CFWE2BWWNSU43L5NQAS3ZCWO/graph.json","events_json":"https://pith.science/api/pith-number/C7CFWE2BWWNSU43L5NQAS3ZCWO/events.json","paper":"https://pith.science/paper/C7CFWE2B"},"agent_actions":{"view_html":"https://pith.science/pith/C7CFWE2BWWNSU43L5NQAS3ZCWO","download_json":"https://pith.science/pith/C7CFWE2BWWNSU43L5NQAS3ZCWO.json","view_paper":"https://pith.science/paper/C7CFWE2B","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2412.00543&json=true","fetch_graph":"https://pith.science/api/pith-number/C7CFWE2BWWNSU43L5NQAS3ZCWO/graph.json","fetch_events":"https://pith.science/api/pith-number/C7CFWE2BWWNSU43L5NQAS3ZCWO/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/C7CFWE2BWWNSU43L5NQAS3ZCWO/action/timestamp_anchor","attest_storage":"https://pith.science/pith/C7CFWE2BWWNSU43L5NQAS3ZCWO/action/storage_attestation","attest_author":"https://pith.science/pith/C7CFWE2BWWNSU43L5NQAS3ZCWO/action/author_attestation","sign_citation":"https://pith.science/pith/C7CFWE2BWWNSU43L5NQAS3ZCWO/action/citation_signature","submit_replication":"https://pith.science/pith/C7CFWE2BWWNSU43L5NQAS3ZCWO/action/replication_record"}},"created_at":"2026-07-05T09:42:43.408983+00:00","updated_at":"2026-07-05T09:42:43.408983+00:00"}