{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:2ZP6NFLW4O53KIRSJCVHPFPSTB","short_pith_number":"pith:2ZP6NFLW","canonical_record":{"source":{"id":"2411.15320","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-11-22T19:28:06Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"a315731942f88bb5a15eb24ab0f439a53dd602d6945db119ae73d402f35d5fe5","abstract_canon_sha256":"0d90c799d9a699c9aae374fa9f3adbb715e0c86d9bcba4be2b7c4b7095ea11aa"},"schema_version":"1.0"},"canonical_sha256":"d65fe69576e3bbb5223248aa7795f2987d2c603c40de7cb2ddb5ffe5eca1c320","source":{"kind":"arxiv","id":"2411.15320","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.15320","created_at":"2026-07-05T09:39:33Z"},{"alias_kind":"arxiv_version","alias_value":"2411.15320v1","created_at":"2026-07-05T09:39:33Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.15320","created_at":"2026-07-05T09:39:33Z"},{"alias_kind":"pith_short_12","alias_value":"2ZP6NFLW4O53","created_at":"2026-07-05T09:39:33Z"},{"alias_kind":"pith_short_16","alias_value":"2ZP6NFLW4O53KIRS","created_at":"2026-07-05T09:39:33Z"},{"alias_kind":"pith_short_8","alias_value":"2ZP6NFLW","created_at":"2026-07-05T09:39:33Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:2ZP6NFLW4O53KIRSJCVHPFPSTB","target":"record","payload":{"canonical_record":{"source":{"id":"2411.15320","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-11-22T19:28:06Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"a315731942f88bb5a15eb24ab0f439a53dd602d6945db119ae73d402f35d5fe5","abstract_canon_sha256":"0d90c799d9a699c9aae374fa9f3adbb715e0c86d9bcba4be2b7c4b7095ea11aa"},"schema_version":"1.0"},"canonical_sha256":"d65fe69576e3bbb5223248aa7795f2987d2c603c40de7cb2ddb5ffe5eca1c320","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:39:33.060573Z","signature_b64":"XypXL4Oiu8U1fDYN6ns43KxtCt0gCIPNBWnQrNoA+Y82pPJ1K1ggsDBCx8oNzUi7sMZZ3dNSBowdCHmrIjp1Aw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d65fe69576e3bbb5223248aa7795f2987d2c603c40de7cb2ddb5ffe5eca1c320","last_reissued_at":"2026-07-05T09:39:33.060074Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:39:33.060074Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2411.15320","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:39:33Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"qAK0bQ/3X8yFvGf3uGJVz9sRSYHTmIw+M1ymL06kC2Ao8S8U/x/dWOSnK9E26OMLCdYqfTAykeTCc9REry+gCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-13T01:32:28.275767Z"},"content_sha256":"ad5ceb76717276c3261225ed633fa1c6ed8de505d3f4cc6a4c13a1b2ed2efe51","schema_version":"1.0","event_id":"sha256:ad5ceb76717276c3261225ed633fa1c6ed8de505d3f4cc6a4c13a1b2ed2efe51"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:2ZP6NFLW4O53KIRSJCVHPFPSTB","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Ashish Khetan, Gerald Friedland, Sanjiv Das, Vishaal Kapoor, Xin Huang, Yueying Cui","submitted_at":"2024-11-22T19:28:06Z","abstract_excerpt":"We propose PPLqa, an easy to compute, language independent, information-theoretic metric to measure the quality of responses of generative Large Language Models (LLMs) in an unsupervised way, without requiring ground truth annotations or human supervision. The method and metric enables users to rank generative language models for quality of responses, so as to make a selection of the best model for a given task. Our single metric assesses LLMs with an approach that subsumes, but is not explicitly based on, coherence and fluency (quality of writing) and relevance and consistency (appropriatenes"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.15320","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2411.15320/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:39:33Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"uQAoY4rywX4Jep8CDrrknaicSOJO7aooDkGVR8ePoao2p9IFe3DmUVM4TNYNen9m9ryR6groPBBkoMCNdfaEAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-13T01:32:28.276298Z"},"content_sha256":"a206b6b128b163f2cb0aeaa8296949268c0dff40ce0c741ce8208de99345eea2","schema_version":"1.0","event_id":"sha256:a206b6b128b163f2cb0aeaa8296949268c0dff40ce0c741ce8208de99345eea2"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/2ZP6NFLW4O53KIRSJCVHPFPSTB/bundle.json","state_url":"https://pith.science/pith/2ZP6NFLW4O53KIRSJCVHPFPSTB/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/2ZP6NFLW4O53KIRSJCVHPFPSTB/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-13T01:32:28Z","links":{"resolver":"https://pith.science/pith/2ZP6NFLW4O53KIRSJCVHPFPSTB","bundle":"https://pith.science/pith/2ZP6NFLW4O53KIRSJCVHPFPSTB/bundle.json","state":"https://pith.science/pith/2ZP6NFLW4O53KIRSJCVHPFPSTB/state.json","well_known_bundle":"https://pith.science/.well-known/pith/2ZP6NFLW4O53KIRSJCVHPFPSTB/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:2ZP6NFLW4O53KIRSJCVHPFPSTB","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0d90c799d9a699c9aae374fa9f3adbb715e0c86d9bcba4be2b7c4b7095ea11aa","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-11-22T19:28:06Z","title_canon_sha256":"a315731942f88bb5a15eb24ab0f439a53dd602d6945db119ae73d402f35d5fe5"},"schema_version":"1.0","source":{"id":"2411.15320","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.15320","created_at":"2026-07-05T09:39:33Z"},{"alias_kind":"arxiv_version","alias_value":"2411.15320v1","created_at":"2026-07-05T09:39:33Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.15320","created_at":"2026-07-05T09:39:33Z"},{"alias_kind":"pith_short_12","alias_value":"2ZP6NFLW4O53","created_at":"2026-07-05T09:39:33Z"},{"alias_kind":"pith_short_16","alias_value":"2ZP6NFLW4O53KIRS","created_at":"2026-07-05T09:39:33Z"},{"alias_kind":"pith_short_8","alias_value":"2ZP6NFLW","created_at":"2026-07-05T09:39:33Z"}],"graph_snapshots":[{"event_id":"sha256:a206b6b128b163f2cb0aeaa8296949268c0dff40ce0c741ce8208de99345eea2","target":"graph","created_at":"2026-07-05T09:39:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2411.15320/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We propose PPLqa, an easy to compute, language independent, information-theoretic metric to measure the quality of responses of generative Large Language Models (LLMs) in an unsupervised way, without requiring ground truth annotations or human supervision. The method and metric enables users to rank generative language models for quality of responses, so as to make a selection of the best model for a given task. Our single metric assesses LLMs with an approach that subsumes, but is not explicitly based on, coherence and fluency (quality of writing) and relevance and consistency (appropriatenes","authors_text":"Ashish Khetan, Gerald Friedland, Sanjiv Das, Vishaal Kapoor, Xin Huang, Yueying Cui","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-11-22T19:28:06Z","title":"PPLqa: An Unsupervised Information-Theoretic Quality Metric for Comparing Generative Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.15320","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:ad5ceb76717276c3261225ed633fa1c6ed8de505d3f4cc6a4c13a1b2ed2efe51","target":"record","created_at":"2026-07-05T09:39:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0d90c799d9a699c9aae374fa9f3adbb715e0c86d9bcba4be2b7c4b7095ea11aa","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-11-22T19:28:06Z","title_canon_sha256":"a315731942f88bb5a15eb24ab0f439a53dd602d6945db119ae73d402f35d5fe5"},"schema_version":"1.0","source":{"id":"2411.15320","kind":"arxiv","version":1}},"canonical_sha256":"d65fe69576e3bbb5223248aa7795f2987d2c603c40de7cb2ddb5ffe5eca1c320","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"d65fe69576e3bbb5223248aa7795f2987d2c603c40de7cb2ddb5ffe5eca1c320","first_computed_at":"2026-07-05T09:39:33.060074Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:39:33.060074Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"XypXL4Oiu8U1fDYN6ns43KxtCt0gCIPNBWnQrNoA+Y82pPJ1K1ggsDBCx8oNzUi7sMZZ3dNSBowdCHmrIjp1Aw==","signature_status":"signed_v1","signed_at":"2026-07-05T09:39:33.060573Z","signed_message":"canonical_sha256_bytes"},"source_id":"2411.15320","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:ad5ceb76717276c3261225ed633fa1c6ed8de505d3f4cc6a4c13a1b2ed2efe51","sha256:a206b6b128b163f2cb0aeaa8296949268c0dff40ce0c741ce8208de99345eea2"],"state_sha256":"6ea747df22fee5e0d879a985e70fa8a80d43f1a916d26504e944515a8daf86ce"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4aqChAVJ38Nvm+C3dI9I2nIrHU1A18tdsWi4g9DYzihoDN+75f76e9H3109O4SMRQSqrXUJfk0Yv/jqFkowZBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-13T01:32:28.281827Z","bundle_sha256":"cfe6b4a506b62ee4cb6f4bafb4cdbdbdd83ac4b9d13106b62aac88c2bc30d1d6"}}