{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:F3RAQP7CYBZAVLWJVV5PHDUO2R","short_pith_number":"pith:F3RAQP7C","schema_version":"1.0","canonical_sha256":"2ee2083fe2c0720aaec9ad7af38e8ed47dd4fd1738ce82933ce434132da778cf","source":{"kind":"arxiv","id":"2410.00873","version":2},"attestation_state":"computed","paper":{"title":"Aligning Human and LLM Judgments: Insights from EvalAssist on Task-Specific Evaluations and AI-assisted Assessment Strategy Preferences","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.HC","authors_text":"Elizabeth M. Daly, Hyo Jin Do, James M. Johnson, Martin Santillan Cooper, Michael Desmond, Qian Pan, Rahul Nair, Tejaswini Pedapati, Werner Geyer, Zahra Ashktorab","submitted_at":"2024-10-01T17:09:01Z","abstract_excerpt":"Evaluation of large language model (LLM) outputs requires users to make critical judgments about the best outputs across various configurations. This process is costly and takes time given the large amounts of data. LLMs are increasingly used as evaluators to filter training data, evaluate model performance or assist human evaluators with detailed assessments. To support this process, effective front-end tools are critical for evaluation. Two common approaches for using LLMs as evaluators are direct assessment and pairwise comparison. In our study with machine learning practitioners (n=15), ea"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2410.00873","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.HC","submitted_at":"2024-10-01T17:09:01Z","cross_cats_sorted":[],"title_canon_sha256":"ccdd8c7cf38e01adb3b16b38ac56471ce610778c6ccc5d3b5d101cb2c7a9848d","abstract_canon_sha256":"ccd6cf77548df893bda170db2140d5d082df5e42bf40456d3b83f918bdf3b1ea"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:49:31.667970Z","signature_b64":"5/mjE+C0J56Ugvq86tunuL/RjoeXiwxc1Jx+UGGK33t5SafzS1ZZRN39chk60xqZNVctSeqweQ0Y4QJIM0WVAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"2ee2083fe2c0720aaec9ad7af38e8ed47dd4fd1738ce82933ce434132da778cf","last_reissued_at":"2026-07-05T11:49:31.667434Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:49:31.667434Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Aligning Human and LLM Judgments: Insights from EvalAssist on Task-Specific Evaluations and AI-assisted Assessment Strategy Preferences","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.HC","authors_text":"Elizabeth M. Daly, Hyo Jin Do, James M. Johnson, Martin Santillan Cooper, Michael Desmond, Qian Pan, Rahul Nair, Tejaswini Pedapati, Werner Geyer, Zahra Ashktorab","submitted_at":"2024-10-01T17:09:01Z","abstract_excerpt":"Evaluation of large language model (LLM) outputs requires users to make critical judgments about the best outputs across various configurations. This process is costly and takes time given the large amounts of data. LLMs are increasingly used as evaluators to filter training data, evaluate model performance or assist human evaluators with detailed assessments. To support this process, effective front-end tools are critical for evaluation. Two common approaches for using LLMs as evaluators are direct assessment and pairwise comparison. In our study with machine learning practitioners (n=15), ea"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.00873","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.00873/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2410.00873","created_at":"2026-07-05T11:49:31.667505+00:00"},{"alias_kind":"arxiv_version","alias_value":"2410.00873v2","created_at":"2026-07-05T11:49:31.667505+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.00873","created_at":"2026-07-05T11:49:31.667505+00:00"},{"alias_kind":"pith_short_12","alias_value":"F3RAQP7CYBZA","created_at":"2026-07-05T11:49:31.667505+00:00"},{"alias_kind":"pith_short_16","alias_value":"F3RAQP7CYBZAVLWJ","created_at":"2026-07-05T11:49:31.667505+00:00"},{"alias_kind":"pith_short_8","alias_value":"F3RAQP7C","created_at":"2026-07-05T11:49:31.667505+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":3,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2509.11206","citing_title":"Evalet: Evaluating Large Language Models through Functional Fragmentation","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2412.05579","citing_title":"LLMs-as-Judges: A Comprehensive Survey on LLM-based Evaluation Methods","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2604.26679","citing_title":"MultEval: Supporting Collaborative Alignment for LLM-as-a-Judge Evaluation Criteria","ref_index":3,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/F3RAQP7CYBZAVLWJVV5PHDUO2R","json":"https://pith.science/pith/F3RAQP7CYBZAVLWJVV5PHDUO2R.json","graph_json":"https://pith.science/api/pith-number/F3RAQP7CYBZAVLWJVV5PHDUO2R/graph.json","events_json":"https://pith.science/api/pith-number/F3RAQP7CYBZAVLWJVV5PHDUO2R/events.json","paper":"https://pith.science/paper/F3RAQP7C"},"agent_actions":{"view_html":"https://pith.science/pith/F3RAQP7CYBZAVLWJVV5PHDUO2R","download_json":"https://pith.science/pith/F3RAQP7CYBZAVLWJVV5PHDUO2R.json","view_paper":"https://pith.science/paper/F3RAQP7C","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2410.00873&json=true","fetch_graph":"https://pith.science/api/pith-number/F3RAQP7CYBZAVLWJVV5PHDUO2R/graph.json","fetch_events":"https://pith.science/api/pith-number/F3RAQP7CYBZAVLWJVV5PHDUO2R/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/F3RAQP7CYBZAVLWJVV5PHDUO2R/action/timestamp_anchor","attest_storage":"https://pith.science/pith/F3RAQP7CYBZAVLWJVV5PHDUO2R/action/storage_attestation","attest_author":"https://pith.science/pith/F3RAQP7CYBZAVLWJVV5PHDUO2R/action/author_attestation","sign_citation":"https://pith.science/pith/F3RAQP7CYBZAVLWJVV5PHDUO2R/action/citation_signature","submit_replication":"https://pith.science/pith/F3RAQP7CYBZAVLWJVV5PHDUO2R/action/replication_record"}},"created_at":"2026-07-05T11:49:31.667505+00:00","updated_at":"2026-07-05T11:49:31.667505+00:00"}