{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:DF2D2SSSRNXRTUT2PZJT3RZ44Z","short_pith_number":"pith:DF2D2SSS","canonical_record":{"source":{"id":"2508.10014","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-08-06T13:06:15Z","cross_cats_sorted":[],"title_canon_sha256":"a6880cea7ae53de8b276cd58c9a93bea8c60ea98bb27870cedfd0ae1860eff79","abstract_canon_sha256":"ff1a60458690fbbffd9d0cb25cd711bcac595d74508a34d19e577ea06dc2e95d"},"schema_version":"1.0"},"canonical_sha256":"19743d4a528b6f19d27a7e533dc73ce654324e1afdd7fd7d1288a01154143727","source":{"kind":"arxiv","id":"2508.10014","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2508.10014","created_at":"2026-07-05T11:53:29Z"},{"alias_kind":"arxiv_version","alias_value":"2508.10014v1","created_at":"2026-07-05T11:53:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2508.10014","created_at":"2026-07-05T11:53:29Z"},{"alias_kind":"pith_short_12","alias_value":"DF2D2SSSRNXR","created_at":"2026-07-05T11:53:29Z"},{"alias_kind":"pith_short_16","alias_value":"DF2D2SSSRNXRTUT2","created_at":"2026-07-05T11:53:29Z"},{"alias_kind":"pith_short_8","alias_value":"DF2D2SSS","created_at":"2026-07-05T11:53:29Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:DF2D2SSSRNXRTUT2PZJT3RZ44Z","target":"record","payload":{"canonical_record":{"source":{"id":"2508.10014","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-08-06T13:06:15Z","cross_cats_sorted":[],"title_canon_sha256":"a6880cea7ae53de8b276cd58c9a93bea8c60ea98bb27870cedfd0ae1860eff79","abstract_canon_sha256":"ff1a60458690fbbffd9d0cb25cd711bcac595d74508a34d19e577ea06dc2e95d"},"schema_version":"1.0"},"canonical_sha256":"19743d4a528b6f19d27a7e533dc73ce654324e1afdd7fd7d1288a01154143727","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:53:29.261669Z","signature_b64":"Luusbaxbj163kjJn7BLJJrGX9yENGGjSql6X5F+7bowlG12s+H5YpiHqGyUIY4nP0dutI+1rHSUNu0y4KqgmBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"19743d4a528b6f19d27a7e533dc73ce654324e1afdd7fd7d1288a01154143727","last_reissued_at":"2026-07-05T11:53:29.261194Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:53:29.261194Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2508.10014","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:53:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"NY8DLZkAzWak/+zEuz2JQtuBAKSOiQfiZa9adYwCi7SCBc2G2Ap3dZOWp5Pyhrk8A+y1pnl4shaPkht1WgtBCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T03:50:34.059701Z"},"content_sha256":"ad0a7ec3aaf0ce22fe52e8260900038766eb03f301a1c8a426568e701294501f","schema_version":"1.0","event_id":"sha256:ad0a7ec3aaf0ce22fe52e8260900038766eb03f301a1c8a426568e701294501f"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:DF2D2SSSRNXRTUT2PZJT3RZ44Z","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"PersonaEval: Are LLM Evaluators Human Enough to Judge Role-Play?","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Dequan Wang, Jialing Zhang, Jin Gao, Lingfeng Zhou, Mohan Jiang","submitted_at":"2025-08-06T13:06:15Z","abstract_excerpt":"Current role-play studies often rely on unvalidated LLM-as-a-judge paradigms, which may fail to reflect how humans perceive role fidelity. A key prerequisite for human-aligned evaluation is role identification, the ability to recognize who is speaking based on dialogue context. We argue that any meaningful judgment of role-playing quality (how well a character is played) fundamentally depends on first correctly attributing words and actions to the correct persona (who is speaking). We present PersonaEval, the first benchmark designed to test whether LLM evaluators can reliably identify human r"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2508.10014","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2508.10014/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:53:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"2XUHdeiJD44LEQB//ay9aWCEYaG3LoLyURu8fp067N1Qv/PkTY7Mx2AYH8tN6iiFsqxfLl1/M1U/hPEhWkLqAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T03:50:34.060074Z"},"content_sha256":"f0bad5e22c14af06928a8c6f046847a24f5e7391d0b031b7a4e6ccdcfb1f1b8f","schema_version":"1.0","event_id":"sha256:f0bad5e22c14af06928a8c6f046847a24f5e7391d0b031b7a4e6ccdcfb1f1b8f"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/DF2D2SSSRNXRTUT2PZJT3RZ44Z/bundle.json","state_url":"https://pith.science/pith/DF2D2SSSRNXRTUT2PZJT3RZ44Z/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/DF2D2SSSRNXRTUT2PZJT3RZ44Z/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T03:50:34Z","links":{"resolver":"https://pith.science/pith/DF2D2SSSRNXRTUT2PZJT3RZ44Z","bundle":"https://pith.science/pith/DF2D2SSSRNXRTUT2PZJT3RZ44Z/bundle.json","state":"https://pith.science/pith/DF2D2SSSRNXRTUT2PZJT3RZ44Z/state.json","well_known_bundle":"https://pith.science/.well-known/pith/DF2D2SSSRNXRTUT2PZJT3RZ44Z/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:DF2D2SSSRNXRTUT2PZJT3RZ44Z","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"ff1a60458690fbbffd9d0cb25cd711bcac595d74508a34d19e577ea06dc2e95d","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-08-06T13:06:15Z","title_canon_sha256":"a6880cea7ae53de8b276cd58c9a93bea8c60ea98bb27870cedfd0ae1860eff79"},"schema_version":"1.0","source":{"id":"2508.10014","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2508.10014","created_at":"2026-07-05T11:53:29Z"},{"alias_kind":"arxiv_version","alias_value":"2508.10014v1","created_at":"2026-07-05T11:53:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2508.10014","created_at":"2026-07-05T11:53:29Z"},{"alias_kind":"pith_short_12","alias_value":"DF2D2SSSRNXR","created_at":"2026-07-05T11:53:29Z"},{"alias_kind":"pith_short_16","alias_value":"DF2D2SSSRNXRTUT2","created_at":"2026-07-05T11:53:29Z"},{"alias_kind":"pith_short_8","alias_value":"DF2D2SSS","created_at":"2026-07-05T11:53:29Z"}],"graph_snapshots":[{"event_id":"sha256:f0bad5e22c14af06928a8c6f046847a24f5e7391d0b031b7a4e6ccdcfb1f1b8f","target":"graph","created_at":"2026-07-05T11:53:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2508.10014/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Current role-play studies often rely on unvalidated LLM-as-a-judge paradigms, which may fail to reflect how humans perceive role fidelity. A key prerequisite for human-aligned evaluation is role identification, the ability to recognize who is speaking based on dialogue context. We argue that any meaningful judgment of role-playing quality (how well a character is played) fundamentally depends on first correctly attributing words and actions to the correct persona (who is speaking). We present PersonaEval, the first benchmark designed to test whether LLM evaluators can reliably identify human r","authors_text":"Dequan Wang, Jialing Zhang, Jin Gao, Lingfeng Zhou, Mohan Jiang","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-08-06T13:06:15Z","title":"PersonaEval: Are LLM Evaluators Human Enough to Judge Role-Play?"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2508.10014","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:ad0a7ec3aaf0ce22fe52e8260900038766eb03f301a1c8a426568e701294501f","target":"record","created_at":"2026-07-05T11:53:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"ff1a60458690fbbffd9d0cb25cd711bcac595d74508a34d19e577ea06dc2e95d","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-08-06T13:06:15Z","title_canon_sha256":"a6880cea7ae53de8b276cd58c9a93bea8c60ea98bb27870cedfd0ae1860eff79"},"schema_version":"1.0","source":{"id":"2508.10014","kind":"arxiv","version":1}},"canonical_sha256":"19743d4a528b6f19d27a7e533dc73ce654324e1afdd7fd7d1288a01154143727","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"19743d4a528b6f19d27a7e533dc73ce654324e1afdd7fd7d1288a01154143727","first_computed_at":"2026-07-05T11:53:29.261194Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:53:29.261194Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Luusbaxbj163kjJn7BLJJrGX9yENGGjSql6X5F+7bowlG12s+H5YpiHqGyUIY4nP0dutI+1rHSUNu0y4KqgmBg==","signature_status":"signed_v1","signed_at":"2026-07-05T11:53:29.261669Z","signed_message":"canonical_sha256_bytes"},"source_id":"2508.10014","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:ad0a7ec3aaf0ce22fe52e8260900038766eb03f301a1c8a426568e701294501f","sha256:f0bad5e22c14af06928a8c6f046847a24f5e7391d0b031b7a4e6ccdcfb1f1b8f"],"state_sha256":"42ce7e0ba3b406fd360c6ff17e5497a680035ee5f6b2705e1c552ff3560e0c71"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"rpTaNnmhcadQh2ZNArnShzNqLwaTLTiiz+WU3JRTYzBXvPW2C79WJX1PgKGi0fs9B0vjCOMOdkP9bhM6cg1SDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T03:50:34.062421Z","bundle_sha256":"c95652856e8b6a2e043ec023343035c6cf0fe76cdf3146e742c55c1f65f02fa8"}}