{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:5UHH7F2SS4G5OC3SOP3WXZ4N2G","short_pith_number":"pith:5UHH7F2S","canonical_record":{"source":{"id":"2406.18403","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-26T14:56:13Z","cross_cats_sorted":[],"title_canon_sha256":"7fbf834f830fea0fcd14b0265258a361185cbcb357440656136358bb638f6e27","abstract_canon_sha256":"b6fdb8abe8782f8d0e350301a1ed31fdc97aae6e93b3d1d766b3be2ae8c37e4a"},"schema_version":"1.0"},"canonical_sha256":"ed0e7f9752970dd70b7273f76be78dd1a0302bd1640fb692da86e0ebdb103a39","source":{"kind":"arxiv","id":"2406.18403","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.18403","created_at":"2026-07-05T11:13:56Z"},{"alias_kind":"arxiv_version","alias_value":"2406.18403v3","created_at":"2026-07-05T11:13:56Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.18403","created_at":"2026-07-05T11:13:56Z"},{"alias_kind":"pith_short_12","alias_value":"5UHH7F2SS4G5","created_at":"2026-07-05T11:13:56Z"},{"alias_kind":"pith_short_16","alias_value":"5UHH7F2SS4G5OC3S","created_at":"2026-07-05T11:13:56Z"},{"alias_kind":"pith_short_8","alias_value":"5UHH7F2S","created_at":"2026-07-05T11:13:56Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:5UHH7F2SS4G5OC3SOP3WXZ4N2G","target":"record","payload":{"canonical_record":{"source":{"id":"2406.18403","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-26T14:56:13Z","cross_cats_sorted":[],"title_canon_sha256":"7fbf834f830fea0fcd14b0265258a361185cbcb357440656136358bb638f6e27","abstract_canon_sha256":"b6fdb8abe8782f8d0e350301a1ed31fdc97aae6e93b3d1d766b3be2ae8c37e4a"},"schema_version":"1.0"},"canonical_sha256":"ed0e7f9752970dd70b7273f76be78dd1a0302bd1640fb692da86e0ebdb103a39","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:13:56.588637Z","signature_b64":"62rU8KKouNYY4woIWau6feU1PasIo8USjhvLBK5Moj/CoqPDvMO/VNcz3L2eli0YNUxv4hc6oqe5IFBc4JfaDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ed0e7f9752970dd70b7273f76be78dd1a0302bd1640fb692da86e0ebdb103a39","last_reissued_at":"2026-07-05T11:13:56.588127Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:13:56.588127Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.18403","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:13:56Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"G728veXFH9Fn4hdOovUsycbH5Qt4HEbFE6A0LgfkUl78E9nLxYte/7JA7czBY8MoM3pESOq7FVjXrayKmPrjAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-02T23:07:13.236969Z"},"content_sha256":"53b499a769c5844ec994879a3be98fe8a45b11bae3c121e60560f763af0a0885","schema_version":"1.0","event_id":"sha256:53b499a769c5844ec994879a3be98fe8a45b11bae3c121e60560f763af0a0885"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:5UHH7F2SS4G5OC3SOP3WXZ4N2G","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"LLMs instead of Human Judges? A Large Scale Empirical Study across 20 NLP Evaluation Tasks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Aditya K Surikuchi, Albert Gatt, Alberto Testoni, Alessandro Suglia, Alexander Koller, Andr\\'e F. T. Martins, Anna Bavaresco, Barbara Plank, David Schlangen, Desmond Elliott, Ece Takmaz, Esam Ghaleb, Leonardo Bertolazzi, Mario Giulianelli, Michael Hanna, Philipp Mondorf, Raffaella Bernardi, Raquel Fern\\'andez, Sandro Pezzelle, Vera Neplenbroek","submitted_at":"2024-06-26T14:56:13Z","abstract_excerpt":"There is an increasing trend towards evaluating NLP models with LLMs instead of human judgments, raising questions about the validity of these evaluations, as well as their reproducibility in the case of proprietary models. We provide JUDGE-BENCH, an extensible collection of 20 NLP datasets with human annotations covering a broad range of evaluated properties and types of data, and comprehensively evaluate 11 current LLMs, covering both open-weight and proprietary models, for their ability to replicate the annotations. Our evaluations show substantial variance across models and datasets. Model"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.18403","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.18403/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:13:56Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"SOdIR8Xc7LAu1O6npvUv6brrNk91oTy9IEEaU1/53UPWi2jil1U9CBmnDwqBI7wocp5uIlAb4MlsLjvCKcR2Dw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-02T23:07:13.237366Z"},"content_sha256":"99648fe0bc0968dff5cb2b826f869185c82ac6f5e50c9f305656c34441959c60","schema_version":"1.0","event_id":"sha256:99648fe0bc0968dff5cb2b826f869185c82ac6f5e50c9f305656c34441959c60"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/5UHH7F2SS4G5OC3SOP3WXZ4N2G/bundle.json","state_url":"https://pith.science/pith/5UHH7F2SS4G5OC3SOP3WXZ4N2G/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/5UHH7F2SS4G5OC3SOP3WXZ4N2G/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-02T23:07:13Z","links":{"resolver":"https://pith.science/pith/5UHH7F2SS4G5OC3SOP3WXZ4N2G","bundle":"https://pith.science/pith/5UHH7F2SS4G5OC3SOP3WXZ4N2G/bundle.json","state":"https://pith.science/pith/5UHH7F2SS4G5OC3SOP3WXZ4N2G/state.json","well_known_bundle":"https://pith.science/.well-known/pith/5UHH7F2SS4G5OC3SOP3WXZ4N2G/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:5UHH7F2SS4G5OC3SOP3WXZ4N2G","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"b6fdb8abe8782f8d0e350301a1ed31fdc97aae6e93b3d1d766b3be2ae8c37e4a","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-26T14:56:13Z","title_canon_sha256":"7fbf834f830fea0fcd14b0265258a361185cbcb357440656136358bb638f6e27"},"schema_version":"1.0","source":{"id":"2406.18403","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.18403","created_at":"2026-07-05T11:13:56Z"},{"alias_kind":"arxiv_version","alias_value":"2406.18403v3","created_at":"2026-07-05T11:13:56Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.18403","created_at":"2026-07-05T11:13:56Z"},{"alias_kind":"pith_short_12","alias_value":"5UHH7F2SS4G5","created_at":"2026-07-05T11:13:56Z"},{"alias_kind":"pith_short_16","alias_value":"5UHH7F2SS4G5OC3S","created_at":"2026-07-05T11:13:56Z"},{"alias_kind":"pith_short_8","alias_value":"5UHH7F2S","created_at":"2026-07-05T11:13:56Z"}],"graph_snapshots":[{"event_id":"sha256:99648fe0bc0968dff5cb2b826f869185c82ac6f5e50c9f305656c34441959c60","target":"graph","created_at":"2026-07-05T11:13:56Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.18403/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"There is an increasing trend towards evaluating NLP models with LLMs instead of human judgments, raising questions about the validity of these evaluations, as well as their reproducibility in the case of proprietary models. We provide JUDGE-BENCH, an extensible collection of 20 NLP datasets with human annotations covering a broad range of evaluated properties and types of data, and comprehensively evaluate 11 current LLMs, covering both open-weight and proprietary models, for their ability to replicate the annotations. Our evaluations show substantial variance across models and datasets. Model","authors_text":"Aditya K Surikuchi, Albert Gatt, Alberto Testoni, Alessandro Suglia, Alexander Koller, Andr\\'e F. T. Martins, Anna Bavaresco, Barbara Plank, David Schlangen, Desmond Elliott, Ece Takmaz, Esam Ghaleb, Leonardo Bertolazzi, Mario Giulianelli, Michael Hanna, Philipp Mondorf, Raffaella Bernardi, Raquel Fern\\'andez, Sandro Pezzelle, Vera Neplenbroek","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-26T14:56:13Z","title":"LLMs instead of Human Judges? A Large Scale Empirical Study across 20 NLP Evaluation Tasks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.18403","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:53b499a769c5844ec994879a3be98fe8a45b11bae3c121e60560f763af0a0885","target":"record","created_at":"2026-07-05T11:13:56Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"b6fdb8abe8782f8d0e350301a1ed31fdc97aae6e93b3d1d766b3be2ae8c37e4a","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-26T14:56:13Z","title_canon_sha256":"7fbf834f830fea0fcd14b0265258a361185cbcb357440656136358bb638f6e27"},"schema_version":"1.0","source":{"id":"2406.18403","kind":"arxiv","version":3}},"canonical_sha256":"ed0e7f9752970dd70b7273f76be78dd1a0302bd1640fb692da86e0ebdb103a39","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"ed0e7f9752970dd70b7273f76be78dd1a0302bd1640fb692da86e0ebdb103a39","first_computed_at":"2026-07-05T11:13:56.588127Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:13:56.588127Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"62rU8KKouNYY4woIWau6feU1PasIo8USjhvLBK5Moj/CoqPDvMO/VNcz3L2eli0YNUxv4hc6oqe5IFBc4JfaDA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:13:56.588637Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.18403","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:53b499a769c5844ec994879a3be98fe8a45b11bae3c121e60560f763af0a0885","sha256:99648fe0bc0968dff5cb2b826f869185c82ac6f5e50c9f305656c34441959c60"],"state_sha256":"9444d2e87b0dba3875e13b6a335630e7b05ab773740794ee9aab4a9ee74ff46e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"uBWKnVO8LC6BSCSEASQRpBAgxL5i4VEo6RVQTFV5sTcgNAB0K0xTEs2yRhoaGczAq+8Z+pZNmnJA7902N76kCg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-02T23:07:13.240066Z","bundle_sha256":"fa7a64d7f0635d966744db237dbce40513d3838fa9a3793375cb5c42de80e7ac"}}