{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:VLGDQ4HJTJY3XNOA5A3FTC5P3L","short_pith_number":"pith:VLGDQ4HJ","schema_version":"1.0","canonical_sha256":"aacc3870e99a71bbb5c0e836598bafdac37186e0491f16fdb256d2baaebd5dc7","source":{"kind":"arxiv","id":"2408.01262","version":5},"attestation_state":"computed","paper":{"title":"RAGEval: Scenario Specific RAG Evaluation Dataset Generation Framework","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.IR"],"primary_cat":"cs.CL","authors_text":"Dingling Xu, Kunlun Zhu, Maosong Sun, Nan Zhang, Ruobing Wang, Shi Yu, Shuo Wang, Xu Han, Yifan Luo, Yishan Li, Yukun Yan, Zhenghao Liu, Zhiyuan Liu","submitted_at":"2024-08-02T13:35:11Z","abstract_excerpt":"Retrieval-Augmented Generation (RAG) is a powerful approach that enables large language models (LLMs) to incorporate external knowledge. However, evaluating the effectiveness of RAG systems in specialized scenarios remains challenging due to the high costs of data construction and the lack of suitable evaluation metrics. This paper introduces RAGEval, a framework designed to assess RAG systems across diverse scenarios by generating high-quality documents, questions, answers, and references through a schema-based pipeline. With a focus on factual accuracy, we propose three novel metrics: Comple"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2408.01262","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-02T13:35:11Z","cross_cats_sorted":["cs.IR"],"title_canon_sha256":"05cccac6e595d42960dc84aba18390e6fa77e49c966d4a7224567f5a91226f8f","abstract_canon_sha256":"263c27d246ef42c189748802e0c06ab25a4cd15ce805f164b727f6aee83dc8d3"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:23:21.781875Z","signature_b64":"VEE4c4PfkxPO5S458m0n0YtkmkIfa6cDH/uynMHLuBl/4LfUYsaVQC5t3ZzFv5KSlYWD9GO+8vbxbnEPsY8QCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"aacc3870e99a71bbb5c0e836598bafdac37186e0491f16fdb256d2baaebd5dc7","last_reissued_at":"2026-07-05T10:23:21.781375Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:23:21.781375Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"RAGEval: Scenario Specific RAG Evaluation Dataset Generation Framework","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.IR"],"primary_cat":"cs.CL","authors_text":"Dingling Xu, Kunlun Zhu, Maosong Sun, Nan Zhang, Ruobing Wang, Shi Yu, Shuo Wang, Xu Han, Yifan Luo, Yishan Li, Yukun Yan, Zhenghao Liu, Zhiyuan Liu","submitted_at":"2024-08-02T13:35:11Z","abstract_excerpt":"Retrieval-Augmented Generation (RAG) is a powerful approach that enables large language models (LLMs) to incorporate external knowledge. However, evaluating the effectiveness of RAG systems in specialized scenarios remains challenging due to the high costs of data construction and the lack of suitable evaluation metrics. This paper introduces RAGEval, a framework designed to assess RAG systems across diverse scenarios by generating high-quality documents, questions, answers, and references through a schema-based pipeline. With a focus on factual accuracy, we propose three novel metrics: Comple"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.01262","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2408.01262/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2408.01262","created_at":"2026-07-05T10:23:21.781428+00:00"},{"alias_kind":"arxiv_version","alias_value":"2408.01262v5","created_at":"2026-07-05T10:23:21.781428+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.01262","created_at":"2026-07-05T10:23:21.781428+00:00"},{"alias_kind":"pith_short_12","alias_value":"VLGDQ4HJTJY3","created_at":"2026-07-05T10:23:21.781428+00:00"},{"alias_kind":"pith_short_16","alias_value":"VLGDQ4HJTJY3XNOA","created_at":"2026-07-05T10:23:21.781428+00:00"},{"alias_kind":"pith_short_8","alias_value":"VLGDQ4HJ","created_at":"2026-07-05T10:23:21.781428+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":3,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2410.10594","citing_title":"VisRAG: Vision-based Retrieval-augmented Generation on Multi-modality Documents","ref_index":33,"is_internal_anchor":false},{"citing_arxiv_id":"2605.08838","citing_title":"Generating Leakage-Free Benchmarks for Robust RAG Evaluation","ref_index":15,"is_internal_anchor":false},{"citing_arxiv_id":"2605.05459","citing_title":"Privacy Without Losing Place: A Paradigm for Private Retrieval in Spatial RAGs","ref_index":6,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/VLGDQ4HJTJY3XNOA5A3FTC5P3L","json":"https://pith.science/pith/VLGDQ4HJTJY3XNOA5A3FTC5P3L.json","graph_json":"https://pith.science/api/pith-number/VLGDQ4HJTJY3XNOA5A3FTC5P3L/graph.json","events_json":"https://pith.science/api/pith-number/VLGDQ4HJTJY3XNOA5A3FTC5P3L/events.json","paper":"https://pith.science/paper/VLGDQ4HJ"},"agent_actions":{"view_html":"https://pith.science/pith/VLGDQ4HJTJY3XNOA5A3FTC5P3L","download_json":"https://pith.science/pith/VLGDQ4HJTJY3XNOA5A3FTC5P3L.json","view_paper":"https://pith.science/paper/VLGDQ4HJ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2408.01262&json=true","fetch_graph":"https://pith.science/api/pith-number/VLGDQ4HJTJY3XNOA5A3FTC5P3L/graph.json","fetch_events":"https://pith.science/api/pith-number/VLGDQ4HJTJY3XNOA5A3FTC5P3L/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/VLGDQ4HJTJY3XNOA5A3FTC5P3L/action/timestamp_anchor","attest_storage":"https://pith.science/pith/VLGDQ4HJTJY3XNOA5A3FTC5P3L/action/storage_attestation","attest_author":"https://pith.science/pith/VLGDQ4HJTJY3XNOA5A3FTC5P3L/action/author_attestation","sign_citation":"https://pith.science/pith/VLGDQ4HJTJY3XNOA5A3FTC5P3L/action/citation_signature","submit_replication":"https://pith.science/pith/VLGDQ4HJTJY3XNOA5A3FTC5P3L/action/replication_record"}},"created_at":"2026-07-05T10:23:21.781428+00:00","updated_at":"2026-07-05T10:23:21.781428+00:00"}