{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:DHJWGSM3XNKWMU6IOXMP4FNV63","short_pith_number":"pith:DHJWGSM3","schema_version":"1.0","canonical_sha256":"19d363499bbb556653c875d8fe15b5f6cf88764f93315a55cb81bb5a838f9a25","source":{"kind":"arxiv","id":"2410.03775","version":3},"attestation_state":"computed","paper":{"title":"Beyond correlation: The Impact of Human Uncertainty in Measuring the Effectiveness of Automatic Evaluation and LLM-as-a-Judge","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.HC","authors_text":"Aparna Elangovan, Dan Roth, Jongwoo Ko, Lei Xu, Ling Liu, Mahsa Elyasi, Sravan Bodapati","submitted_at":"2024-10-03T03:08:29Z","abstract_excerpt":"The effectiveness of automatic evaluation of generative models is typically measured by comparing the labels generated via automation with labels by humans using correlation metrics. However, metrics like Krippendorff's $\\alpha$ and Randolph's $\\kappa$ were originally designed to measure the reliability of human labeling, thus make assumptions about typical human labeling behavior, and these assumptions may not be applicable to machine generated labels. In this paper, we show how *relying on a single aggregate correlation score* can obscure fundamental differences between human labels and thos"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2410.03775","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.HC","submitted_at":"2024-10-03T03:08:29Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"69a2d9134937a08856bc470d8aa800df82b8e5e76da7b107b37952bec63f8ecd","abstract_canon_sha256":"4b1fa5e8f03564214055e61612e9661e00f9a2a5a05c352bbdd4cfe1eba10ff1"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:05:45.203245Z","signature_b64":"0Z40gcURlgTELbKj0Fa6NDvHfNGLIQbfaxHDUfkJ2RIMs0pZgn+KpvptFMUGvLDtGP9BZwr2keaiwqFKpZyCDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"19d363499bbb556653c875d8fe15b5f6cf88764f93315a55cb81bb5a838f9a25","last_reissued_at":"2026-07-05T10:05:45.202715Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:05:45.202715Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Beyond correlation: The Impact of Human Uncertainty in Measuring the Effectiveness of Automatic Evaluation and LLM-as-a-Judge","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.HC","authors_text":"Aparna Elangovan, Dan Roth, Jongwoo Ko, Lei Xu, Ling Liu, Mahsa Elyasi, Sravan Bodapati","submitted_at":"2024-10-03T03:08:29Z","abstract_excerpt":"The effectiveness of automatic evaluation of generative models is typically measured by comparing the labels generated via automation with labels by humans using correlation metrics. However, metrics like Krippendorff's $\\alpha$ and Randolph's $\\kappa$ were originally designed to measure the reliability of human labeling, thus make assumptions about typical human labeling behavior, and these assumptions may not be applicable to machine generated labels. In this paper, we show how *relying on a single aggregate correlation score* can obscure fundamental differences between human labels and thos"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.03775","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.03775/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2410.03775","created_at":"2026-07-05T10:05:45.202807+00:00"},{"alias_kind":"arxiv_version","alias_value":"2410.03775v3","created_at":"2026-07-05T10:05:45.202807+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.03775","created_at":"2026-07-05T10:05:45.202807+00:00"},{"alias_kind":"pith_short_12","alias_value":"DHJWGSM3XNKW","created_at":"2026-07-05T10:05:45.202807+00:00"},{"alias_kind":"pith_short_16","alias_value":"DHJWGSM3XNKWMU6I","created_at":"2026-07-05T10:05:45.202807+00:00"},{"alias_kind":"pith_short_8","alias_value":"DHJWGSM3","created_at":"2026-07-05T10:05:45.202807+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":5,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.09013","citing_title":"Beyond Averages: Evaluating LLMs on Human Survey Replication at the Distributional Level","ref_index":8,"is_internal_anchor":false},{"citing_arxiv_id":"2508.15503","citing_title":"Guidelines for Empirical Studies in Software Engineering involving Large Language Models","ref_index":36,"is_internal_anchor":false},{"citing_arxiv_id":"2508.15503","citing_title":"Guidelines for Empirical Studies in Software Engineering involving Large Language Models","ref_index":36,"is_internal_anchor":false},{"citing_arxiv_id":"2604.15329","citing_title":"Evaluating LLMs as Human Surrogates in Controlled Experiments","ref_index":15,"is_internal_anchor":false},{"citing_arxiv_id":"2412.05579","citing_title":"LLMs-as-Judges: A Comprehensive Survey on LLM-based Evaluation Methods","ref_index":58,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/DHJWGSM3XNKWMU6IOXMP4FNV63","json":"https://pith.science/pith/DHJWGSM3XNKWMU6IOXMP4FNV63.json","graph_json":"https://pith.science/api/pith-number/DHJWGSM3XNKWMU6IOXMP4FNV63/graph.json","events_json":"https://pith.science/api/pith-number/DHJWGSM3XNKWMU6IOXMP4FNV63/events.json","paper":"https://pith.science/paper/DHJWGSM3"},"agent_actions":{"view_html":"https://pith.science/pith/DHJWGSM3XNKWMU6IOXMP4FNV63","download_json":"https://pith.science/pith/DHJWGSM3XNKWMU6IOXMP4FNV63.json","view_paper":"https://pith.science/paper/DHJWGSM3","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2410.03775&json=true","fetch_graph":"https://pith.science/api/pith-number/DHJWGSM3XNKWMU6IOXMP4FNV63/graph.json","fetch_events":"https://pith.science/api/pith-number/DHJWGSM3XNKWMU6IOXMP4FNV63/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/DHJWGSM3XNKWMU6IOXMP4FNV63/action/timestamp_anchor","attest_storage":"https://pith.science/pith/DHJWGSM3XNKWMU6IOXMP4FNV63/action/storage_attestation","attest_author":"https://pith.science/pith/DHJWGSM3XNKWMU6IOXMP4FNV63/action/author_attestation","sign_citation":"https://pith.science/pith/DHJWGSM3XNKWMU6IOXMP4FNV63/action/citation_signature","submit_replication":"https://pith.science/pith/DHJWGSM3XNKWMU6IOXMP4FNV63/action/replication_record"}},"created_at":"2026-07-05T10:05:45.202807+00:00","updated_at":"2026-07-05T10:05:45.202807+00:00"}