{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:456TNFUFWMF7MFYX37W7FPALST","short_pith_number":"pith:456TNFUF","schema_version":"1.0","canonical_sha256":"e77d369685b30bf61717dfedf2bc0b94d8544179a53f44a17fd80e3b1ea75922","source":{"kind":"arxiv","id":"2406.11657","version":1},"attestation_state":"computed","paper":{"title":"Can LLM be a Personalized Judge?","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CY"],"primary_cat":"cs.CL","authors_text":"Nigel Collier, Tiancheng Hu, Yijiang River Dong","submitted_at":"2024-06-17T15:41:30Z","abstract_excerpt":"Ensuring that large language models (LLMs) reflect diverse user values and preferences is crucial as their user bases expand globally. It is therefore encouraging to see the growing interest in LLM personalization within the research community. However, current works often rely on the LLM-as-a-Judge approach for evaluation without thoroughly examining its validity. In this paper, we investigate the reliability of LLM-as-a-Personalized-Judge, asking LLMs to judge user preferences based on personas. Our findings suggest that directly applying LLM-as-a-Personalized-Judge is less reliable than pre"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2406.11657","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-06-17T15:41:30Z","cross_cats_sorted":["cs.CY"],"title_canon_sha256":"7dd3d8300c31ec75745625e2b31c30248613712e5433aaf1ed2014ec5a879971","abstract_canon_sha256":"4fe89b6d5bb65000ec90d559f4fff60efa6e601787894a96889c7772de64f431"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:32:56.032436Z","signature_b64":"0uE96A+E2MAo/P1N2wGOzyFv7/82AyH1PYg08xkZgmLlXDKoB6U4FjYGv7TKMIM5fuNf3LvsfDfkg4xWpDLXBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"e77d369685b30bf61717dfedf2bc0b94d8544179a53f44a17fd80e3b1ea75922","last_reissued_at":"2026-07-05T08:32:56.031959Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:32:56.031959Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Can LLM be a Personalized Judge?","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CY"],"primary_cat":"cs.CL","authors_text":"Nigel Collier, Tiancheng Hu, Yijiang River Dong","submitted_at":"2024-06-17T15:41:30Z","abstract_excerpt":"Ensuring that large language models (LLMs) reflect diverse user values and preferences is crucial as their user bases expand globally. It is therefore encouraging to see the growing interest in LLM personalization within the research community. However, current works often rely on the LLM-as-a-Judge approach for evaluation without thoroughly examining its validity. In this paper, we investigate the reliability of LLM-as-a-Personalized-Judge, asking LLMs to judge user preferences based on personas. Our findings suggest that directly applying LLM-as-a-Personalized-Judge is less reliable than pre"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.11657","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.11657/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2406.11657","created_at":"2026-07-05T08:32:56.032015+00:00"},{"alias_kind":"arxiv_version","alias_value":"2406.11657v1","created_at":"2026-07-05T08:32:56.032015+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.11657","created_at":"2026-07-05T08:32:56.032015+00:00"},{"alias_kind":"pith_short_12","alias_value":"456TNFUFWMF7","created_at":"2026-07-05T08:32:56.032015+00:00"},{"alias_kind":"pith_short_16","alias_value":"456TNFUFWMF7MFYX","created_at":"2026-07-05T08:32:56.032015+00:00"},{"alias_kind":"pith_short_8","alias_value":"456TNFUF","created_at":"2026-07-05T08:32:56.032015+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":8,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2605.27395","citing_title":"Informing AI Policy Assessment using Large-Scale Simulation of Interventions","ref_index":32,"is_internal_anchor":true},{"citing_arxiv_id":"2606.03189","citing_title":"SenseJudge: Human-Centric Preference-Driven Judgment Framework","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2411.15594","citing_title":"A Survey on LLM-as-a-Judge","ref_index":31,"is_internal_anchor":false},{"citing_arxiv_id":"2504.04927","citing_title":"Creating and Evaluating Personas Using Generative AI: A Scoping Review of 81 Articles","ref_index":27,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10367","citing_title":"AgentGR: Semantic-aware Agentic Group Decision-Making Simulator for Group Recommendation","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2412.05579","citing_title":"LLMs-as-Judges: A Comprehensive Survey on LLM-based Evaluation Methods","ref_index":54,"is_internal_anchor":false},{"citing_arxiv_id":"2604.12479","citing_title":"Meet Dynamic Individual Preferences: Resolving Conflicting Human Value with Paired Fine-Tuning","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2604.09876","citing_title":"Efficient Personalization of Generative User Interfaces","ref_index":26,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/456TNFUFWMF7MFYX37W7FPALST","json":"https://pith.science/pith/456TNFUFWMF7MFYX37W7FPALST.json","graph_json":"https://pith.science/api/pith-number/456TNFUFWMF7MFYX37W7FPALST/graph.json","events_json":"https://pith.science/api/pith-number/456TNFUFWMF7MFYX37W7FPALST/events.json","paper":"https://pith.science/paper/456TNFUF"},"agent_actions":{"view_html":"https://pith.science/pith/456TNFUFWMF7MFYX37W7FPALST","download_json":"https://pith.science/pith/456TNFUFWMF7MFYX37W7FPALST.json","view_paper":"https://pith.science/paper/456TNFUF","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2406.11657&json=true","fetch_graph":"https://pith.science/api/pith-number/456TNFUFWMF7MFYX37W7FPALST/graph.json","fetch_events":"https://pith.science/api/pith-number/456TNFUFWMF7MFYX37W7FPALST/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/456TNFUFWMF7MFYX37W7FPALST/action/timestamp_anchor","attest_storage":"https://pith.science/pith/456TNFUFWMF7MFYX37W7FPALST/action/storage_attestation","attest_author":"https://pith.science/pith/456TNFUFWMF7MFYX37W7FPALST/action/author_attestation","sign_citation":"https://pith.science/pith/456TNFUFWMF7MFYX37W7FPALST/action/citation_signature","submit_replication":"https://pith.science/pith/456TNFUFWMF7MFYX37W7FPALST/action/replication_record"}},"created_at":"2026-07-05T08:32:56.032015+00:00","updated_at":"2026-07-05T08:32:56.032015+00:00"}