{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:Y5ZYSTXPRRU57ENSOQG6OP4SC3","short_pith_number":"pith:Y5ZYSTXP","canonical_record":{"source":{"id":"2505.11626","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-16T18:42:04Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"5e9c31953a5b3a295b7fb033eb3564fd7119ca8bc267172174f9e66014ebee74","abstract_canon_sha256":"7d8db034a49f52e57f1d1b62ffbf4dd57d10a919d7b9d8af986f6442f929632c"},"schema_version":"1.0"},"canonical_sha256":"c773894eef8c69df91b2740de73f9216d9830135ea7e42b5716efced58fca6a2","source":{"kind":"arxiv","id":"2505.11626","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.11626","created_at":"2026-07-05T11:15:21Z"},{"alias_kind":"arxiv_version","alias_value":"2505.11626v2","created_at":"2026-07-05T11:15:21Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.11626","created_at":"2026-07-05T11:15:21Z"},{"alias_kind":"pith_short_12","alias_value":"Y5ZYSTXPRRU5","created_at":"2026-07-05T11:15:21Z"},{"alias_kind":"pith_short_16","alias_value":"Y5ZYSTXPRRU57ENS","created_at":"2026-07-05T11:15:21Z"},{"alias_kind":"pith_short_8","alias_value":"Y5ZYSTXP","created_at":"2026-07-05T11:15:21Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:Y5ZYSTXPRRU57ENSOQG6OP4SC3","target":"record","payload":{"canonical_record":{"source":{"id":"2505.11626","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-16T18:42:04Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"5e9c31953a5b3a295b7fb033eb3564fd7119ca8bc267172174f9e66014ebee74","abstract_canon_sha256":"7d8db034a49f52e57f1d1b62ffbf4dd57d10a919d7b9d8af986f6442f929632c"},"schema_version":"1.0"},"canonical_sha256":"c773894eef8c69df91b2740de73f9216d9830135ea7e42b5716efced58fca6a2","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:15:21.030587Z","signature_b64":"dLgO/z9YGkyqFCE3JIIFZzQMJpM/2bHXPHNJpddj5AZVIRtHSOLixfBKIXo+Zu11X0nS5wuGF17egZnzkamrBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c773894eef8c69df91b2740de73f9216d9830135ea7e42b5716efced58fca6a2","last_reissued_at":"2026-07-05T11:15:21.030073Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:15:21.030073Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2505.11626","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:15:21Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"wYAEaoqmeZNBa3ahzMJDL7D1CYd0FVXNtLMLe++09YZx5OcT3whpq/0KflnWDNQKn/k6CS22G3/FSLqXILsDAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T22:10:08.655839Z"},"content_sha256":"c65b5a079a879e57d303ace61a3aae641fe735b3e28fc0e812effa227e4a2039","schema_version":"1.0","event_id":"sha256:c65b5a079a879e57d303ace61a3aae641fe735b3e28fc0e812effa227e4a2039"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:Y5ZYSTXPRRU57ENSOQG6OP4SC3","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"THELMA: Task Based Holistic Evaluation of Large Language Model Applications-RAG Question Answering","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Cibi Chakravarthy Senthilkumar, Jay Roberts, Naumaan Nayyar, Parul Kalra, Rafael Castrillo, Rutu Mulkar, Sujay Gandhi, Udita Patel, Xiaofei Zheng","submitted_at":"2025-05-16T18:42:04Z","abstract_excerpt":"We propose THELMA (Task Based Holistic Evaluation of Large Language Model Applications), a reference free framework for RAG (Retrieval Augmented generation) based question answering (QA) applications. THELMA consist of six interdependent metrics specifically designed for holistic, fine grained evaluation of RAG QA applications. THELMA framework helps developers and application owners evaluate, monitor and improve end to end RAG QA pipelines without requiring labelled sources or reference responses.We also present our findings on the interplay of the proposed THELMA metrics, which can be interp"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.11626","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2505.11626/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:15:21Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"DKxX8Z/TZ9sBQHSR9AGywHg1IrpDMyCEH31ZPlaiV6FM7rmeggCA5Wl4zJo6sHYUAg4C2n/+RJKjCILtCk/NAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T22:10:08.656347Z"},"content_sha256":"25de8939de1a6302511f9523d079e7fce11774d4249ba49cbb24135fb37a26c5","schema_version":"1.0","event_id":"sha256:25de8939de1a6302511f9523d079e7fce11774d4249ba49cbb24135fb37a26c5"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/Y5ZYSTXPRRU57ENSOQG6OP4SC3/bundle.json","state_url":"https://pith.science/pith/Y5ZYSTXPRRU57ENSOQG6OP4SC3/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/Y5ZYSTXPRRU57ENSOQG6OP4SC3/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-21T22:10:08Z","links":{"resolver":"https://pith.science/pith/Y5ZYSTXPRRU57ENSOQG6OP4SC3","bundle":"https://pith.science/pith/Y5ZYSTXPRRU57ENSOQG6OP4SC3/bundle.json","state":"https://pith.science/pith/Y5ZYSTXPRRU57ENSOQG6OP4SC3/state.json","well_known_bundle":"https://pith.science/.well-known/pith/Y5ZYSTXPRRU57ENSOQG6OP4SC3/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:Y5ZYSTXPRRU57ENSOQG6OP4SC3","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7d8db034a49f52e57f1d1b62ffbf4dd57d10a919d7b9d8af986f6442f929632c","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-16T18:42:04Z","title_canon_sha256":"5e9c31953a5b3a295b7fb033eb3564fd7119ca8bc267172174f9e66014ebee74"},"schema_version":"1.0","source":{"id":"2505.11626","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.11626","created_at":"2026-07-05T11:15:21Z"},{"alias_kind":"arxiv_version","alias_value":"2505.11626v2","created_at":"2026-07-05T11:15:21Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.11626","created_at":"2026-07-05T11:15:21Z"},{"alias_kind":"pith_short_12","alias_value":"Y5ZYSTXPRRU5","created_at":"2026-07-05T11:15:21Z"},{"alias_kind":"pith_short_16","alias_value":"Y5ZYSTXPRRU57ENS","created_at":"2026-07-05T11:15:21Z"},{"alias_kind":"pith_short_8","alias_value":"Y5ZYSTXP","created_at":"2026-07-05T11:15:21Z"}],"graph_snapshots":[{"event_id":"sha256:25de8939de1a6302511f9523d079e7fce11774d4249ba49cbb24135fb37a26c5","target":"graph","created_at":"2026-07-05T11:15:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2505.11626/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We propose THELMA (Task Based Holistic Evaluation of Large Language Model Applications), a reference free framework for RAG (Retrieval Augmented generation) based question answering (QA) applications. THELMA consist of six interdependent metrics specifically designed for holistic, fine grained evaluation of RAG QA applications. THELMA framework helps developers and application owners evaluate, monitor and improve end to end RAG QA pipelines without requiring labelled sources or reference responses.We also present our findings on the interplay of the proposed THELMA metrics, which can be interp","authors_text":"Cibi Chakravarthy Senthilkumar, Jay Roberts, Naumaan Nayyar, Parul Kalra, Rafael Castrillo, Rutu Mulkar, Sujay Gandhi, Udita Patel, Xiaofei Zheng","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-16T18:42:04Z","title":"THELMA: Task Based Holistic Evaluation of Large Language Model Applications-RAG Question Answering"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.11626","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:c65b5a079a879e57d303ace61a3aae641fe735b3e28fc0e812effa227e4a2039","target":"record","created_at":"2026-07-05T11:15:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7d8db034a49f52e57f1d1b62ffbf4dd57d10a919d7b9d8af986f6442f929632c","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-16T18:42:04Z","title_canon_sha256":"5e9c31953a5b3a295b7fb033eb3564fd7119ca8bc267172174f9e66014ebee74"},"schema_version":"1.0","source":{"id":"2505.11626","kind":"arxiv","version":2}},"canonical_sha256":"c773894eef8c69df91b2740de73f9216d9830135ea7e42b5716efced58fca6a2","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c773894eef8c69df91b2740de73f9216d9830135ea7e42b5716efced58fca6a2","first_computed_at":"2026-07-05T11:15:21.030073Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:15:21.030073Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"dLgO/z9YGkyqFCE3JIIFZzQMJpM/2bHXPHNJpddj5AZVIRtHSOLixfBKIXo+Zu11X0nS5wuGF17egZnzkamrBA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:15:21.030587Z","signed_message":"canonical_sha256_bytes"},"source_id":"2505.11626","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:c65b5a079a879e57d303ace61a3aae641fe735b3e28fc0e812effa227e4a2039","sha256:25de8939de1a6302511f9523d079e7fce11774d4249ba49cbb24135fb37a26c5"],"state_sha256":"b736ede954ec633fa86e2ac41d9383fc77587678978ed9b2bb0e227f3f96edb3"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"3+qxHH/L8uc0oenFXi+xReArJYpEN5PAGSNHfOR2CuPvdUqvlwR+D3SPXxTYL5fU+fV47K+E29Frwg8Lq87jDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-21T22:10:08.661484Z","bundle_sha256":"4f5183eee1082047be6940d9fb5b1c255665730667747d18505070a8077be361"}}