{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:V26VZM4JY2DZ4P6GZPWNJQHIMR","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"21dfc0fa27f7e92fa90f4ebc5fc206c49949742502cf2b0bf07da6bdfbb3bace","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.HC","submitted_at":"2024-10-26T20:35:14Z","title_canon_sha256":"3b74af65934b2ea1d05a46b3f55a093df872392dc1bb4f4aefab536816fd0b91"},"schema_version":"1.0","source":{"id":"2410.20266","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.20266","created_at":"2026-07-05T09:26:41Z"},{"alias_kind":"arxiv_version","alias_value":"2410.20266v1","created_at":"2026-07-05T09:26:41Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.20266","created_at":"2026-07-05T09:26:41Z"},{"alias_kind":"pith_short_12","alias_value":"V26VZM4JY2DZ","created_at":"2026-07-05T09:26:41Z"},{"alias_kind":"pith_short_16","alias_value":"V26VZM4JY2DZ4P6G","created_at":"2026-07-05T09:26:41Z"},{"alias_kind":"pith_short_8","alias_value":"V26VZM4J","created_at":"2026-07-05T09:26:41Z"}],"graph_snapshots":[{"event_id":"sha256:5a76021dada1cc446cce8c9d30be30dd9d4c39832bca7189eb65598bfac2ca54","target":"graph","created_at":"2026-07-05T09:26:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.20266/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The potential of using Large Language Models (LLMs) themselves to evaluate LLM outputs offers a promising method for assessing model performance across various contexts. Previous research indicates that LLM-as-a-judge exhibits a strong correlation with human judges in the context of general instruction following. However, for instructions that require specialized knowledge, the validity of using LLMs as judges remains uncertain. In our study, we applied a mixed-methods approach, conducting pairwise comparisons in which both subject matter experts (SMEs) and LLMs evaluated outputs from domain-s","authors_text":"Annalisa Szymanski, Heather A. Eicher-Miller, Meng Jiang, Noah Ziems, Ronald A. Metoyer, Toby Jia-Jun Li","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.HC","submitted_at":"2024-10-26T20:35:14Z","title":"Limitations of the LLM-as-a-Judge Approach for Evaluating LLM Outputs in Expert Knowledge Tasks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.20266","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:06562d3b885cf34aed4f7c134f0d6d60253c2ba49a48b7f36d20d3d4bb334d3d","target":"record","created_at":"2026-07-05T09:26:41Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"21dfc0fa27f7e92fa90f4ebc5fc206c49949742502cf2b0bf07da6bdfbb3bace","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.HC","submitted_at":"2024-10-26T20:35:14Z","title_canon_sha256":"3b74af65934b2ea1d05a46b3f55a093df872392dc1bb4f4aefab536816fd0b91"},"schema_version":"1.0","source":{"id":"2410.20266","kind":"arxiv","version":1}},"canonical_sha256":"aebd5cb389c6879e3fc6cbecd4c0e8644e80febe0afc09986c61c1f31286a97f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"aebd5cb389c6879e3fc6cbecd4c0e8644e80febe0afc09986c61c1f31286a97f","first_computed_at":"2026-07-05T09:26:41.713787Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:26:41.713787Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"PhOG5MK7H49Z4afNWgcpcqQZccf7J33qiJ2ILeyXBBuXy15KLoIdXMHW7wY5w8GdXo+i5SfQPiXsvPUt7kjzAw==","signature_status":"signed_v1","signed_at":"2026-07-05T09:26:41.714286Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.20266","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:06562d3b885cf34aed4f7c134f0d6d60253c2ba49a48b7f36d20d3d4bb334d3d","sha256:5a76021dada1cc446cce8c9d30be30dd9d4c39832bca7189eb65598bfac2ca54"],"state_sha256":"299d062da4999f08755dcdb5f8f50bf29c40fbef8114b260a8a7b237146a2ff9"}