{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:KMG54PZ5RU67KS3NDLJFOY5A6A","short_pith_number":"pith:KMG54PZ5","canonical_record":{"source":{"id":"2408.13006","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-23T11:49:01Z","cross_cats_sorted":[],"title_canon_sha256":"e5a936264659838f91168c66b010c063503fc1bea4ea2dbdc90c4b73efa50a1a","abstract_canon_sha256":"e3954681621c4e312c5ecf6c67aa11df51e7e93838d7909c87dc2829c6cde122"},"schema_version":"1.0"},"canonical_sha256":"530dde3f3d8d3df54b6d1ad25763a0f01d108a6b257d4a0982d703c883c389ed","source":{"kind":"arxiv","id":"2408.13006","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.13006","created_at":"2026-07-05T10:41:14Z"},{"alias_kind":"arxiv_version","alias_value":"2408.13006v2","created_at":"2026-07-05T10:41:14Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.13006","created_at":"2026-07-05T10:41:14Z"},{"alias_kind":"pith_short_12","alias_value":"KMG54PZ5RU67","created_at":"2026-07-05T10:41:14Z"},{"alias_kind":"pith_short_16","alias_value":"KMG54PZ5RU67KS3N","created_at":"2026-07-05T10:41:14Z"},{"alias_kind":"pith_short_8","alias_value":"KMG54PZ5","created_at":"2026-07-05T10:41:14Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:KMG54PZ5RU67KS3NDLJFOY5A6A","target":"record","payload":{"canonical_record":{"source":{"id":"2408.13006","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-23T11:49:01Z","cross_cats_sorted":[],"title_canon_sha256":"e5a936264659838f91168c66b010c063503fc1bea4ea2dbdc90c4b73efa50a1a","abstract_canon_sha256":"e3954681621c4e312c5ecf6c67aa11df51e7e93838d7909c87dc2829c6cde122"},"schema_version":"1.0"},"canonical_sha256":"530dde3f3d8d3df54b6d1ad25763a0f01d108a6b257d4a0982d703c883c389ed","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:41:14.639865Z","signature_b64":"ATC2wc4g8Xqxuo6mGDcbee6Jd+mQwpfiNIF3rxcqMNMMB67vxoEUiccVXXF3apApYbaDUpUSlUrfmjeI3hGjAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"530dde3f3d8d3df54b6d1ad25763a0f01d108a6b257d4a0982d703c883c389ed","last_reissued_at":"2026-07-05T10:41:14.639362Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:41:14.639362Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2408.13006","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:41:14Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"/aOSSZrB2V/8FVLKA6ngZK2FCMtvGWIwjlYZrG5nXrxaP59kw+NVbcNeg6iOxn1YcV/4jDdsxPCyxGMEePNfCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-22T20:49:49.246122Z"},"content_sha256":"b375abd9b096e31182e7b071728e0cc6a18a03815e9ce99d1ff7f996fa8d9674","schema_version":"1.0","event_id":"sha256:b375abd9b096e31182e7b071728e0cc6a18a03815e9ce99d1ff7f996fa8d9674"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:KMG54PZ5RU67KS3NDLJFOY5A6A","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Systematic Evaluation of LLM-as-a-Judge in LLM Alignment Tasks: Explainable Metrics and Diverse Prompt Templates","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Andy Wong, Fei Liu, Hui Wei, Jingyang Lin, Mei Han, Shenghua He, Tian Xia","submitted_at":"2024-08-23T11:49:01Z","abstract_excerpt":"LLM-as-a-Judge has been widely applied to evaluate and compare different LLM alignmnet approaches (e.g., RLHF and DPO). However, concerns regarding its reliability have emerged, due to LLM judges' biases and inconsistent decision-making. Previous research has developed evaluation frameworks to assess reliability of LLM judges and their alignment with human preferences. However, the employed evaluation metrics often lack adequate explainability and fail to address LLM internal inconsistency. Additionally, existing studies inadequately explore the impact of various prompt templates when applying"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.13006","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2408.13006/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:41:14Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"gyH38iaNIns5FLdk/vdHsBIc+B2YwOxwErsVoiinKycY0WLPgITWi1sTZopEtCdJH3XEioGoamxMrtf9NncsDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-22T20:49:49.246687Z"},"content_sha256":"f9b3dd5946fcd6f4f4506c51c9347357b6c739b60372900d731eb11ad052811b","schema_version":"1.0","event_id":"sha256:f9b3dd5946fcd6f4f4506c51c9347357b6c739b60372900d731eb11ad052811b"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/KMG54PZ5RU67KS3NDLJFOY5A6A/bundle.json","state_url":"https://pith.science/pith/KMG54PZ5RU67KS3NDLJFOY5A6A/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/KMG54PZ5RU67KS3NDLJFOY5A6A/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-22T20:49:49Z","links":{"resolver":"https://pith.science/pith/KMG54PZ5RU67KS3NDLJFOY5A6A","bundle":"https://pith.science/pith/KMG54PZ5RU67KS3NDLJFOY5A6A/bundle.json","state":"https://pith.science/pith/KMG54PZ5RU67KS3NDLJFOY5A6A/state.json","well_known_bundle":"https://pith.science/.well-known/pith/KMG54PZ5RU67KS3NDLJFOY5A6A/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:KMG54PZ5RU67KS3NDLJFOY5A6A","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"e3954681621c4e312c5ecf6c67aa11df51e7e93838d7909c87dc2829c6cde122","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-23T11:49:01Z","title_canon_sha256":"e5a936264659838f91168c66b010c063503fc1bea4ea2dbdc90c4b73efa50a1a"},"schema_version":"1.0","source":{"id":"2408.13006","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.13006","created_at":"2026-07-05T10:41:14Z"},{"alias_kind":"arxiv_version","alias_value":"2408.13006v2","created_at":"2026-07-05T10:41:14Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.13006","created_at":"2026-07-05T10:41:14Z"},{"alias_kind":"pith_short_12","alias_value":"KMG54PZ5RU67","created_at":"2026-07-05T10:41:14Z"},{"alias_kind":"pith_short_16","alias_value":"KMG54PZ5RU67KS3N","created_at":"2026-07-05T10:41:14Z"},{"alias_kind":"pith_short_8","alias_value":"KMG54PZ5","created_at":"2026-07-05T10:41:14Z"}],"graph_snapshots":[{"event_id":"sha256:f9b3dd5946fcd6f4f4506c51c9347357b6c739b60372900d731eb11ad052811b","target":"graph","created_at":"2026-07-05T10:41:14Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2408.13006/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"LLM-as-a-Judge has been widely applied to evaluate and compare different LLM alignmnet approaches (e.g., RLHF and DPO). However, concerns regarding its reliability have emerged, due to LLM judges' biases and inconsistent decision-making. Previous research has developed evaluation frameworks to assess reliability of LLM judges and their alignment with human preferences. However, the employed evaluation metrics often lack adequate explainability and fail to address LLM internal inconsistency. Additionally, existing studies inadequately explore the impact of various prompt templates when applying","authors_text":"Andy Wong, Fei Liu, Hui Wei, Jingyang Lin, Mei Han, Shenghua He, Tian Xia","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-23T11:49:01Z","title":"Systematic Evaluation of LLM-as-a-Judge in LLM Alignment Tasks: Explainable Metrics and Diverse Prompt Templates"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.13006","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b375abd9b096e31182e7b071728e0cc6a18a03815e9ce99d1ff7f996fa8d9674","target":"record","created_at":"2026-07-05T10:41:14Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"e3954681621c4e312c5ecf6c67aa11df51e7e93838d7909c87dc2829c6cde122","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-23T11:49:01Z","title_canon_sha256":"e5a936264659838f91168c66b010c063503fc1bea4ea2dbdc90c4b73efa50a1a"},"schema_version":"1.0","source":{"id":"2408.13006","kind":"arxiv","version":2}},"canonical_sha256":"530dde3f3d8d3df54b6d1ad25763a0f01d108a6b257d4a0982d703c883c389ed","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"530dde3f3d8d3df54b6d1ad25763a0f01d108a6b257d4a0982d703c883c389ed","first_computed_at":"2026-07-05T10:41:14.639362Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:41:14.639362Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"ATC2wc4g8Xqxuo6mGDcbee6Jd+mQwpfiNIF3rxcqMNMMB67vxoEUiccVXXF3apApYbaDUpUSlUrfmjeI3hGjAg==","signature_status":"signed_v1","signed_at":"2026-07-05T10:41:14.639865Z","signed_message":"canonical_sha256_bytes"},"source_id":"2408.13006","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b375abd9b096e31182e7b071728e0cc6a18a03815e9ce99d1ff7f996fa8d9674","sha256:f9b3dd5946fcd6f4f4506c51c9347357b6c739b60372900d731eb11ad052811b"],"state_sha256":"a6250a94517f5cdffe4a1d7d4fb9e8a183da784ee8baeccc6fee5c044ca5f906"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"FHhffHgRl3/dRoQWrLeE6An0i2ycT4vMXnJ598D028FCcLKtHWEKghUQ9Sv3/34okiKoPH5EijT0NCHxZHIoCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-22T20:49:49.251805Z","bundle_sha256":"df537f295f34c8d36fa4d7b897f8dad5b30bccc6f858084bf08c6a96b30cc27d"}}