{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:QUSVK6UFCGZJ673V3TPAUQ7KAI","short_pith_number":"pith:QUSVK6UF","canonical_record":{"source":{"id":"2508.18076","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-08-25T14:43:10Z","cross_cats_sorted":[],"title_canon_sha256":"6440b8172c8ecde77618761fb3473b63cebb499adcfc90c771d1d1a22e8d6df6","abstract_canon_sha256":"7bea651b6aa1d553d51952b31ebc143cc87014d3184394253ccfbb76880309d2"},"schema_version":"1.0"},"canonical_sha256":"8525557a8511b29f7f75dcde0a43ea02071777681155e240a4633a1c3e377f71","source":{"kind":"arxiv","id":"2508.18076","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2508.18076","created_at":"2026-07-05T12:00:29Z"},{"alias_kind":"arxiv_version","alias_value":"2508.18076v2","created_at":"2026-07-05T12:00:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2508.18076","created_at":"2026-07-05T12:00:29Z"},{"alias_kind":"pith_short_12","alias_value":"QUSVK6UFCGZJ","created_at":"2026-07-05T12:00:29Z"},{"alias_kind":"pith_short_16","alias_value":"QUSVK6UFCGZJ673V","created_at":"2026-07-05T12:00:29Z"},{"alias_kind":"pith_short_8","alias_value":"QUSVK6UF","created_at":"2026-07-05T12:00:29Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:QUSVK6UFCGZJ673V3TPAUQ7KAI","target":"record","payload":{"canonical_record":{"source":{"id":"2508.18076","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-08-25T14:43:10Z","cross_cats_sorted":[],"title_canon_sha256":"6440b8172c8ecde77618761fb3473b63cebb499adcfc90c771d1d1a22e8d6df6","abstract_canon_sha256":"7bea651b6aa1d553d51952b31ebc143cc87014d3184394253ccfbb76880309d2"},"schema_version":"1.0"},"canonical_sha256":"8525557a8511b29f7f75dcde0a43ea02071777681155e240a4633a1c3e377f71","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T12:00:29.167826Z","signature_b64":"dTQcH3cXNMwIDzPQ5zLBs1pVfCYoBq4WE6i7xkamfw7CR8rZ+9B2K08qE8KZK2JwH2bLZWLXyqCYsvLrbODVBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8525557a8511b29f7f75dcde0a43ea02071777681155e240a4633a1c3e377f71","last_reissued_at":"2026-07-05T12:00:29.167204Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T12:00:29.167204Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2508.18076","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T12:00:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"/LIIxbs2xJYyyIrums/neaoPYnEflw9qrP6+9s2+/dScKsVuGZk0J0hlTdMt7M3HtAP8DFfwINjktiaZ1CaHAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T13:54:44.565734Z"},"content_sha256":"2d660f48893c581e43383e970b56a42056c7cfa5c3208e160a4429a8430262f4","schema_version":"1.0","event_id":"sha256:2d660f48893c581e43383e970b56a42056c7cfa5c3208e160a4429a8430262f4"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:QUSVK6UFCGZJ673V3TPAUQ7KAI","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Neither Valid nor Reliable? Investigating the Use of LLMs as Judges","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Golnoosh Farnadi, Jackie Chi Kit Cheung, Khaoula Chehbouni, Mohammed Haddou","submitted_at":"2025-08-25T14:43:10Z","abstract_excerpt":"Evaluating natural language generation (NLG) systems remains a core challenge of natural language processing (NLP), further complicated by the rise of large language models (LLMs) that aims to be general-purpose. Recently, large language models as judges (LLJs) have emerged as a promising alternative to traditional metrics, but their validity remains underexplored. This position paper argues that the current enthusiasm around LLJs may be premature, as their adoption has outpaced rigorous scrutiny of their reliability and validity as evaluators. Drawing on measurement theory from the social sci"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2508.18076","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2508.18076/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T12:00:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"KIjxQk5cMl+NV7eYDL6YkUtO0fZTH0jnI6zD84sx1siqXMh4c1hvZMqDy89GQ/HIDRf7CDUz0y15cTE8fw6DAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T13:54:44.566108Z"},"content_sha256":"99a552f86b88ed22cf79944d03bebe1d48ecc28036b8c3afd476cb188bf7daf0","schema_version":"1.0","event_id":"sha256:99a552f86b88ed22cf79944d03bebe1d48ecc28036b8c3afd476cb188bf7daf0"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/QUSVK6UFCGZJ673V3TPAUQ7KAI/bundle.json","state_url":"https://pith.science/pith/QUSVK6UFCGZJ673V3TPAUQ7KAI/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/QUSVK6UFCGZJ673V3TPAUQ7KAI/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-06T13:54:44Z","links":{"resolver":"https://pith.science/pith/QUSVK6UFCGZJ673V3TPAUQ7KAI","bundle":"https://pith.science/pith/QUSVK6UFCGZJ673V3TPAUQ7KAI/bundle.json","state":"https://pith.science/pith/QUSVK6UFCGZJ673V3TPAUQ7KAI/state.json","well_known_bundle":"https://pith.science/.well-known/pith/QUSVK6UFCGZJ673V3TPAUQ7KAI/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:QUSVK6UFCGZJ673V3TPAUQ7KAI","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7bea651b6aa1d553d51952b31ebc143cc87014d3184394253ccfbb76880309d2","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-08-25T14:43:10Z","title_canon_sha256":"6440b8172c8ecde77618761fb3473b63cebb499adcfc90c771d1d1a22e8d6df6"},"schema_version":"1.0","source":{"id":"2508.18076","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2508.18076","created_at":"2026-07-05T12:00:29Z"},{"alias_kind":"arxiv_version","alias_value":"2508.18076v2","created_at":"2026-07-05T12:00:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2508.18076","created_at":"2026-07-05T12:00:29Z"},{"alias_kind":"pith_short_12","alias_value":"QUSVK6UFCGZJ","created_at":"2026-07-05T12:00:29Z"},{"alias_kind":"pith_short_16","alias_value":"QUSVK6UFCGZJ673V","created_at":"2026-07-05T12:00:29Z"},{"alias_kind":"pith_short_8","alias_value":"QUSVK6UF","created_at":"2026-07-05T12:00:29Z"}],"graph_snapshots":[{"event_id":"sha256:99a552f86b88ed22cf79944d03bebe1d48ecc28036b8c3afd476cb188bf7daf0","target":"graph","created_at":"2026-07-05T12:00:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2508.18076/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Evaluating natural language generation (NLG) systems remains a core challenge of natural language processing (NLP), further complicated by the rise of large language models (LLMs) that aims to be general-purpose. Recently, large language models as judges (LLJs) have emerged as a promising alternative to traditional metrics, but their validity remains underexplored. This position paper argues that the current enthusiasm around LLJs may be premature, as their adoption has outpaced rigorous scrutiny of their reliability and validity as evaluators. Drawing on measurement theory from the social sci","authors_text":"Golnoosh Farnadi, Jackie Chi Kit Cheung, Khaoula Chehbouni, Mohammed Haddou","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-08-25T14:43:10Z","title":"Neither Valid nor Reliable? Investigating the Use of LLMs as Judges"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2508.18076","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:2d660f48893c581e43383e970b56a42056c7cfa5c3208e160a4429a8430262f4","target":"record","created_at":"2026-07-05T12:00:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7bea651b6aa1d553d51952b31ebc143cc87014d3184394253ccfbb76880309d2","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-08-25T14:43:10Z","title_canon_sha256":"6440b8172c8ecde77618761fb3473b63cebb499adcfc90c771d1d1a22e8d6df6"},"schema_version":"1.0","source":{"id":"2508.18076","kind":"arxiv","version":2}},"canonical_sha256":"8525557a8511b29f7f75dcde0a43ea02071777681155e240a4633a1c3e377f71","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"8525557a8511b29f7f75dcde0a43ea02071777681155e240a4633a1c3e377f71","first_computed_at":"2026-07-05T12:00:29.167204Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T12:00:29.167204Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"dTQcH3cXNMwIDzPQ5zLBs1pVfCYoBq4WE6i7xkamfw7CR8rZ+9B2K08qE8KZK2JwH2bLZWLXyqCYsvLrbODVBA==","signature_status":"signed_v1","signed_at":"2026-07-05T12:00:29.167826Z","signed_message":"canonical_sha256_bytes"},"source_id":"2508.18076","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:2d660f48893c581e43383e970b56a42056c7cfa5c3208e160a4429a8430262f4","sha256:99a552f86b88ed22cf79944d03bebe1d48ecc28036b8c3afd476cb188bf7daf0"],"state_sha256":"dedc9bf5c8ae4ed99b6a6fd3619c22ed12fd536dcdbf84e8265a2c108971f39c"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"RrOMwtHz8f0ACjhO9FUEdXDgqjRNfK/exluX52f5UzFLdndE0yK+b08L6qW13nMaCMJ7SaBwmTHxzAB7b+44BQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-06T13:54:44.568346Z","bundle_sha256":"8633fcb9a9136c9aedc0f9e57b70a86c9363a0a1e59583a1fec2048674ca8096"}}