{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:OXM3VLQ2HKNL3ED5IMJTJIXTLJ","short_pith_number":"pith:OXM3VLQ2","canonical_record":{"source":{"id":"2406.12624","kind":"arxiv","version":6},"metadata":{"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.CL","submitted_at":"2024-06-18T13:49:54Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"84dd766717e1c69f2702919e663d515f1b0b05d69b3aab4c660b0ca348d2e99e","abstract_canon_sha256":"998a61e7698b7970653d26b39cff7c2fc0e238244b9eaf0cb0dd9a72138d0fa0"},"schema_version":"1.0"},"canonical_sha256":"75d9baae1a3a9abd907d431334a2f35a7e1ae34e470644d1936cc76c5a573109","source":{"kind":"arxiv","id":"2406.12624","version":6},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.12624","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"arxiv_version","alias_value":"2406.12624v6","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.12624","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"pith_short_12","alias_value":"OXM3VLQ2HKNL","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"pith_short_16","alias_value":"OXM3VLQ2HKNL3ED5","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"pith_short_8","alias_value":"OXM3VLQ2","created_at":"2026-07-05T11:54:40Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:OXM3VLQ2HKNL3ED5IMJTJIXTLJ","target":"record","payload":{"canonical_record":{"source":{"id":"2406.12624","kind":"arxiv","version":6},"metadata":{"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.CL","submitted_at":"2024-06-18T13:49:54Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"84dd766717e1c69f2702919e663d515f1b0b05d69b3aab4c660b0ca348d2e99e","abstract_canon_sha256":"998a61e7698b7970653d26b39cff7c2fc0e238244b9eaf0cb0dd9a72138d0fa0"},"schema_version":"1.0"},"canonical_sha256":"75d9baae1a3a9abd907d431334a2f35a7e1ae34e470644d1936cc76c5a573109","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:54:40.224464Z","signature_b64":"FlDYoENNnRQnNkVl7dvjHes7JRy5HRO+++lm02zMb0lv8n+e3oA7M3HG/mtfzLxUdEQufMRsnWI72WLB/JPtBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"75d9baae1a3a9abd907d431334a2f35a7e1ae34e470644d1936cc76c5a573109","last_reissued_at":"2026-07-05T11:54:40.224004Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:54:40.224004Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.12624","source_version":6,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:54:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"hCTrfRBi6fF4w9+QyRj2Z7y00zau1cHBtF2x+NM3BVQ6dp+Q/jjpN99eU74bm3dzQbBVGIe3eIINBW8kh+ZrBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-02T17:58:18.157353Z"},"content_sha256":"404388ea156e92b87cd257b43ef4405981a75df1d3e5040f92c5817c25c91408","schema_version":"1.0","event_id":"sha256:404388ea156e92b87cd257b43ef4405981a75df1d3e5040f92c5817c25c91408"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:OXM3VLQ2HKNL3ED5IMJTJIXTLJ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Judging the Judges: Evaluating Alignment and Vulnerabilities in LLMs-as-Judges","license":"http://creativecommons.org/publicdomain/zero/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Aman Singh Thakur, Dieuwke Hupkes, Kartik Choudhary, Sankaran Vaidyanathan, Venkat Srinik Ramayapally","submitted_at":"2024-06-18T13:49:54Z","abstract_excerpt":"Offering a promising solution to the scalability challenges associated with human evaluation, the LLM-as-a-judge paradigm is rapidly gaining traction as an approach to evaluating large language models (LLMs). However, there are still many open questions about the strengths and weaknesses of this paradigm, and what potential biases it may hold. In this paper, we present a comprehensive study of the performance of various LLMs acting as judges, focusing on a clean scenario in which inter-human agreement is high. Investigating thirteen judge models of different model sizes and families, judging a"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.12624","kind":"arxiv","version":6},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.12624/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:54:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"UN6Kf2QPqjtcFywDhswJowWI/A6i2tV/Rqc2a31AlVRUtO5rKBPz3zn+cRQ+8qKMJfQYjsDr/9tTVrXC+bifCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-02T17:58:18.157863Z"},"content_sha256":"9d28f2ad11b54df648e0285b0442b71a9edba018e17f0509ecda9a0a3aa2da77","schema_version":"1.0","event_id":"sha256:9d28f2ad11b54df648e0285b0442b71a9edba018e17f0509ecda9a0a3aa2da77"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/OXM3VLQ2HKNL3ED5IMJTJIXTLJ/bundle.json","state_url":"https://pith.science/pith/OXM3VLQ2HKNL3ED5IMJTJIXTLJ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/OXM3VLQ2HKNL3ED5IMJTJIXTLJ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-02T17:58:18Z","links":{"resolver":"https://pith.science/pith/OXM3VLQ2HKNL3ED5IMJTJIXTLJ","bundle":"https://pith.science/pith/OXM3VLQ2HKNL3ED5IMJTJIXTLJ/bundle.json","state":"https://pith.science/pith/OXM3VLQ2HKNL3ED5IMJTJIXTLJ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/OXM3VLQ2HKNL3ED5IMJTJIXTLJ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:OXM3VLQ2HKNL3ED5IMJTJIXTLJ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"998a61e7698b7970653d26b39cff7c2fc0e238244b9eaf0cb0dd9a72138d0fa0","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.CL","submitted_at":"2024-06-18T13:49:54Z","title_canon_sha256":"84dd766717e1c69f2702919e663d515f1b0b05d69b3aab4c660b0ca348d2e99e"},"schema_version":"1.0","source":{"id":"2406.12624","kind":"arxiv","version":6}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.12624","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"arxiv_version","alias_value":"2406.12624v6","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.12624","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"pith_short_12","alias_value":"OXM3VLQ2HKNL","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"pith_short_16","alias_value":"OXM3VLQ2HKNL3ED5","created_at":"2026-07-05T11:54:40Z"},{"alias_kind":"pith_short_8","alias_value":"OXM3VLQ2","created_at":"2026-07-05T11:54:40Z"}],"graph_snapshots":[{"event_id":"sha256:9d28f2ad11b54df648e0285b0442b71a9edba018e17f0509ecda9a0a3aa2da77","target":"graph","created_at":"2026-07-05T11:54:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.12624/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Offering a promising solution to the scalability challenges associated with human evaluation, the LLM-as-a-judge paradigm is rapidly gaining traction as an approach to evaluating large language models (LLMs). However, there are still many open questions about the strengths and weaknesses of this paradigm, and what potential biases it may hold. In this paper, we present a comprehensive study of the performance of various LLMs acting as judges, focusing on a clean scenario in which inter-human agreement is high. Investigating thirteen judge models of different model sizes and families, judging a","authors_text":"Aman Singh Thakur, Dieuwke Hupkes, Kartik Choudhary, Sankaran Vaidyanathan, Venkat Srinik Ramayapally","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.CL","submitted_at":"2024-06-18T13:49:54Z","title":"Judging the Judges: Evaluating Alignment and Vulnerabilities in LLMs-as-Judges"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.12624","kind":"arxiv","version":6},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:404388ea156e92b87cd257b43ef4405981a75df1d3e5040f92c5817c25c91408","target":"record","created_at":"2026-07-05T11:54:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"998a61e7698b7970653d26b39cff7c2fc0e238244b9eaf0cb0dd9a72138d0fa0","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.CL","submitted_at":"2024-06-18T13:49:54Z","title_canon_sha256":"84dd766717e1c69f2702919e663d515f1b0b05d69b3aab4c660b0ca348d2e99e"},"schema_version":"1.0","source":{"id":"2406.12624","kind":"arxiv","version":6}},"canonical_sha256":"75d9baae1a3a9abd907d431334a2f35a7e1ae34e470644d1936cc76c5a573109","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"75d9baae1a3a9abd907d431334a2f35a7e1ae34e470644d1936cc76c5a573109","first_computed_at":"2026-07-05T11:54:40.224004Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:54:40.224004Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"FlDYoENNnRQnNkVl7dvjHes7JRy5HRO+++lm02zMb0lv8n+e3oA7M3HG/mtfzLxUdEQufMRsnWI72WLB/JPtBg==","signature_status":"signed_v1","signed_at":"2026-07-05T11:54:40.224464Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.12624","source_kind":"arxiv","source_version":6}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:404388ea156e92b87cd257b43ef4405981a75df1d3e5040f92c5817c25c91408","sha256:9d28f2ad11b54df648e0285b0442b71a9edba018e17f0509ecda9a0a3aa2da77"],"state_sha256":"87b7d56ba1eac7c8c9c2c0f7bff76b244c7e01dba406b7965ede8c959c75431f"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CgH3nWJxF00Md8JsYSsBc1BZunW+SFYw4bA6NX3hryGyKgbzm7s3s3hgBT1aPsWgdpO/2W1O5oH/7UZP7ljEDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-02T17:58:18.161792Z","bundle_sha256":"851b9eac245ace9c79b8eaf2ed523575dbfc99cd4d9cc2a1cecefbb6f346d261"}}