{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:I7TMMKYYLB4VEHIMVGHLQZTSJ3","short_pith_number":"pith:I7TMMKYY","schema_version":"1.0","canonical_sha256":"47e6c62b185879521d0ca98eb866724ecbdb21584a2920c93f14a16d5feb0b59","source":{"kind":"arxiv","id":"2502.12501","version":2},"attestation_state":"computed","paper":{"title":"Crowd Comparative Reasoning: Unlocking Comprehensive Evaluations for LLM-as-a-Judge","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Chen Ma, Chuhan Wu, Fuyuan Lyu, Liangyou Li, Lifeng Shang, Qiyuan Zhang, Ruiming Tang, Xin Jiang, Yasheng Wang, Yufei Wang, Yuxin Jiang","submitted_at":"2025-02-18T03:31:06Z","abstract_excerpt":"LLM-as-a-Judge, which generates chain-of-thought (CoT) judgments, has become a widely adopted auto-evaluation method. However, its reliability is compromised by the CoT reasoning's inability to capture comprehensive and deeper details, often leading to incomplete outcomes. Existing methods mainly rely on majority voting or criteria expansion, which is insufficient to address the limitation in CoT. We propose Crowd-based Comparative Evaluation, which introduces additional crowd responses to compare with the candidate responses, thereby exposing deeper and more comprehensive details within the c"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2502.12501","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-18T03:31:06Z","cross_cats_sorted":[],"title_canon_sha256":"be173c77d959d3d96a861eb7f773d13b5f593c752f0c93d329c8da3101376309","abstract_canon_sha256":"cfa28a7c00df579c43e00441818b4e3b074f139661798d4d982314596e959384"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:45:17.287056Z","signature_b64":"SLippN6aBSQ3S7UT8vUWPzT13VNA6nZjRt4ttp+oCOZMWw7kx8waDYIlIAVakRJSPPco5CDHHdncYhegnFltCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"47e6c62b185879521d0ca98eb866724ecbdb21584a2920c93f14a16d5feb0b59","last_reissued_at":"2026-07-05T10:45:17.286524Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:45:17.286524Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Crowd Comparative Reasoning: Unlocking Comprehensive Evaluations for LLM-as-a-Judge","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Chen Ma, Chuhan Wu, Fuyuan Lyu, Liangyou Li, Lifeng Shang, Qiyuan Zhang, Ruiming Tang, Xin Jiang, Yasheng Wang, Yufei Wang, Yuxin Jiang","submitted_at":"2025-02-18T03:31:06Z","abstract_excerpt":"LLM-as-a-Judge, which generates chain-of-thought (CoT) judgments, has become a widely adopted auto-evaluation method. However, its reliability is compromised by the CoT reasoning's inability to capture comprehensive and deeper details, often leading to incomplete outcomes. Existing methods mainly rely on majority voting or criteria expansion, which is insufficient to address the limitation in CoT. We propose Crowd-based Comparative Evaluation, which introduces additional crowd responses to compare with the candidate responses, thereby exposing deeper and more comprehensive details within the c"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.12501","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.12501/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2502.12501","created_at":"2026-07-05T10:45:17.286603+00:00"},{"alias_kind":"arxiv_version","alias_value":"2502.12501v2","created_at":"2026-07-05T10:45:17.286603+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.12501","created_at":"2026-07-05T10:45:17.286603+00:00"},{"alias_kind":"pith_short_12","alias_value":"I7TMMKYYLB4V","created_at":"2026-07-05T10:45:17.286603+00:00"},{"alias_kind":"pith_short_16","alias_value":"I7TMMKYYLB4VEHIM","created_at":"2026-07-05T10:45:17.286603+00:00"},{"alias_kind":"pith_short_8","alias_value":"I7TMMKYY","created_at":"2026-07-05T10:45:17.286603+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2411.15594","citing_title":"A Survey on LLM-as-a-Judge","ref_index":210,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/I7TMMKYYLB4VEHIMVGHLQZTSJ3","json":"https://pith.science/pith/I7TMMKYYLB4VEHIMVGHLQZTSJ3.json","graph_json":"https://pith.science/api/pith-number/I7TMMKYYLB4VEHIMVGHLQZTSJ3/graph.json","events_json":"https://pith.science/api/pith-number/I7TMMKYYLB4VEHIMVGHLQZTSJ3/events.json","paper":"https://pith.science/paper/I7TMMKYY"},"agent_actions":{"view_html":"https://pith.science/pith/I7TMMKYYLB4VEHIMVGHLQZTSJ3","download_json":"https://pith.science/pith/I7TMMKYYLB4VEHIMVGHLQZTSJ3.json","view_paper":"https://pith.science/paper/I7TMMKYY","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2502.12501&json=true","fetch_graph":"https://pith.science/api/pith-number/I7TMMKYYLB4VEHIMVGHLQZTSJ3/graph.json","fetch_events":"https://pith.science/api/pith-number/I7TMMKYYLB4VEHIMVGHLQZTSJ3/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/I7TMMKYYLB4VEHIMVGHLQZTSJ3/action/timestamp_anchor","attest_storage":"https://pith.science/pith/I7TMMKYYLB4VEHIMVGHLQZTSJ3/action/storage_attestation","attest_author":"https://pith.science/pith/I7TMMKYYLB4VEHIMVGHLQZTSJ3/action/author_attestation","sign_citation":"https://pith.science/pith/I7TMMKYYLB4VEHIMVGHLQZTSJ3/action/citation_signature","submit_replication":"https://pith.science/pith/I7TMMKYYLB4VEHIMVGHLQZTSJ3/action/replication_record"}},"created_at":"2026-07-05T10:45:17.286603+00:00","updated_at":"2026-07-05T10:45:17.286603+00:00"}