{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:HDB5SHCWVIOWB3XA42IVPPWPY2","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"d64b7b50644362dda10f1c6ea5898282231014fe497f6de16774379cda815b1e","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-23T00:02:21Z","title_canon_sha256":"9bd13244e707a0373b700417d65fe81c7fdf7d6599b0413d989db4432aee12f3"},"schema_version":"1.0","source":{"id":"2411.15387","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.15387","created_at":"2026-07-05T09:47:57Z"},{"alias_kind":"arxiv_version","alias_value":"2411.15387v2","created_at":"2026-07-05T09:47:57Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.15387","created_at":"2026-07-05T09:47:57Z"},{"alias_kind":"pith_short_12","alias_value":"HDB5SHCWVIOW","created_at":"2026-07-05T09:47:57Z"},{"alias_kind":"pith_short_16","alias_value":"HDB5SHCWVIOWB3XA","created_at":"2026-07-05T09:47:57Z"},{"alias_kind":"pith_short_8","alias_value":"HDB5SHCW","created_at":"2026-07-05T09:47:57Z"}],"graph_snapshots":[{"event_id":"sha256:70e685bc147ddf6eaabec33c9d5fe76b4eb1eb3706f60206e57a447305688301","target":"graph","created_at":"2026-07-05T09:47:57Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2411.15387/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"As LLMs continue to become more powerful and versatile, human evaluation has quickly become intractable at scale and reliance on automatic metrics has become the norm. Recently, it has been shown that LLMs are themselves state-of-the-art evaluators for many tasks. These Autoraters are typically designed so that they generalize to new systems and test sets. In practice, however, evaluation is performed on a small set of fixed, canonical test sets, which are carefully curated to measure certain capabilities of interest and are not changed frequently. In this work, we design a method which specia","authors_text":"Dan Deutsch, Geza Kovacs, Juraj Juraska, Mara Finkelstein, Markus Freitag, Parker Riley","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-23T00:02:21Z","title":"From Jack of All Trades to Master of One: Specializing LLM-based Autoraters to a Test Set"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.15387","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:e516902dce495009643fe92e9584fabdcac07f9b8fc547faaa9c629e7d3fc5d2","target":"record","created_at":"2026-07-05T09:47:57Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"d64b7b50644362dda10f1c6ea5898282231014fe497f6de16774379cda815b1e","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-11-23T00:02:21Z","title_canon_sha256":"9bd13244e707a0373b700417d65fe81c7fdf7d6599b0413d989db4432aee12f3"},"schema_version":"1.0","source":{"id":"2411.15387","kind":"arxiv","version":2}},"canonical_sha256":"38c3d91c56aa1d60eee0e69157becfc69bf560e709adb1139007970bbb7a0ceb","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"38c3d91c56aa1d60eee0e69157becfc69bf560e709adb1139007970bbb7a0ceb","first_computed_at":"2026-07-05T09:47:57.595862Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:47:57.595862Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"mQ3i7yaEoktjuktVyrCKJCUzdB85OO0TIk36cVcV4OSP6TcLI8danI5ANjKp2mX4Q14MQQ1hq9YCgcjGRi52Bw==","signature_status":"signed_v1","signed_at":"2026-07-05T09:47:57.596365Z","signed_message":"canonical_sha256_bytes"},"source_id":"2411.15387","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:e516902dce495009643fe92e9584fabdcac07f9b8fc547faaa9c629e7d3fc5d2","sha256:70e685bc147ddf6eaabec33c9d5fe76b4eb1eb3706f60206e57a447305688301"],"state_sha256":"2fdf588842e48c19b7657918275a7ffd9268b82e2bfb113085a402099f1276e9"}