{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:NF2SXP6ISKI5XHBB2WUX3HQMEU","short_pith_number":"pith:NF2SXP6I","canonical_record":{"source":{"id":"2411.19710","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2024-11-29T13:57:07Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"c96b28ae5bafd7e19d1c3fd7a7c65c2c681d2eaf7ffc31f5d54f016a04d41209","abstract_canon_sha256":"a01067a9082019d6f081d7d30278534e8b25dff2104322b7c031c9f8110242a3"},"schema_version":"1.0"},"canonical_sha256":"69752bbfc89291db9c21d5a97d9e0c251f75b7b077025453d4b7be4ac30088a3","source":{"kind":"arxiv","id":"2411.19710","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.19710","created_at":"2026-07-05T09:42:07Z"},{"alias_kind":"arxiv_version","alias_value":"2411.19710v1","created_at":"2026-07-05T09:42:07Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.19710","created_at":"2026-07-05T09:42:07Z"},{"alias_kind":"pith_short_12","alias_value":"NF2SXP6ISKI5","created_at":"2026-07-05T09:42:07Z"},{"alias_kind":"pith_short_16","alias_value":"NF2SXP6ISKI5XHBB","created_at":"2026-07-05T09:42:07Z"},{"alias_kind":"pith_short_8","alias_value":"NF2SXP6I","created_at":"2026-07-05T09:42:07Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:NF2SXP6ISKI5XHBB2WUX3HQMEU","target":"record","payload":{"canonical_record":{"source":{"id":"2411.19710","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2024-11-29T13:57:07Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"c96b28ae5bafd7e19d1c3fd7a7c65c2c681d2eaf7ffc31f5d54f016a04d41209","abstract_canon_sha256":"a01067a9082019d6f081d7d30278534e8b25dff2104322b7c031c9f8110242a3"},"schema_version":"1.0"},"canonical_sha256":"69752bbfc89291db9c21d5a97d9e0c251f75b7b077025453d4b7be4ac30088a3","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:42:07.097538Z","signature_b64":"H8DeK0By/GHlE9qaN4Ax6ZDiFHZMSFlr5QJxRk2dTk9AYmOqDEr0Z6LsRaMoD2AO4L4GKWHSsddP5mHY2ajfCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"69752bbfc89291db9c21d5a97d9e0c251f75b7b077025453d4b7be4ac30088a3","last_reissued_at":"2026-07-05T09:42:07.097018Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:42:07.097018Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2411.19710","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:42:07Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"LMaNWppsAidyzpDdOqxq2SGQoaB3zeuv+HVkyGN5mcXBYf+kEjSlHc8FCrCXoxoWj0BKwQdw+/F9TZL5MOTtCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-15T21:59:17.358678Z"},"content_sha256":"6843eb82302727b95b452b0fb6bf10e1104e9718b51f77e59d28f84154627d4f","schema_version":"1.0","event_id":"sha256:6843eb82302727b95b452b0fb6bf10e1104e9718b51f77e59d28f84154627d4f"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:NF2SXP6ISKI5XHBB2WUX3HQMEU","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Know Your RAG: Dataset Taxonomy and Generation Strategies for Evaluating RAG Systems","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.IR","authors_text":"(2) IBM Research Zurich), Cesar Berrospi (2), Lokesh Mishra (2), Michele Dolfi (2), Panagiotis Vagenas (2) ((1) IBM Research Paris-Saclay, Peter Staar (2), Rafael Teixeira de Lima (1), Shubham Gupta (1)","submitted_at":"2024-11-29T13:57:07Z","abstract_excerpt":"Retrieval Augmented Generation (RAG) systems are a widespread application of Large Language Models (LLMs) in the industry. While many tools exist empowering developers to build their own systems, measuring their performance locally, with datasets reflective of the system's use cases, is a technological challenge. Solutions to this problem range from non-specific and cheap (most public datasets) to specific and costly (generating data from local documents). In this paper, we show that using public question and answer (Q&A) datasets to assess retrieval performance can lead to non-optimal systems"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.19710","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2411.19710/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:42:07Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"xkAhru2pveiWhnzYbBpAuvmP9A+s5FXy8XZBbFVB5px1AK60OiVeaznt/O6F38jtUXubUJS8TaOjuWRgzBaDDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-15T21:59:17.359181Z"},"content_sha256":"7f49a67f9e3f9e4177c1f5abf658f87f0954dbdf01a610bbfdb3600541df85b8","schema_version":"1.0","event_id":"sha256:7f49a67f9e3f9e4177c1f5abf658f87f0954dbdf01a610bbfdb3600541df85b8"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/NF2SXP6ISKI5XHBB2WUX3HQMEU/bundle.json","state_url":"https://pith.science/pith/NF2SXP6ISKI5XHBB2WUX3HQMEU/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/NF2SXP6ISKI5XHBB2WUX3HQMEU/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-15T21:59:17Z","links":{"resolver":"https://pith.science/pith/NF2SXP6ISKI5XHBB2WUX3HQMEU","bundle":"https://pith.science/pith/NF2SXP6ISKI5XHBB2WUX3HQMEU/bundle.json","state":"https://pith.science/pith/NF2SXP6ISKI5XHBB2WUX3HQMEU/state.json","well_known_bundle":"https://pith.science/.well-known/pith/NF2SXP6ISKI5XHBB2WUX3HQMEU/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:NF2SXP6ISKI5XHBB2WUX3HQMEU","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"a01067a9082019d6f081d7d30278534e8b25dff2104322b7c031c9f8110242a3","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2024-11-29T13:57:07Z","title_canon_sha256":"c96b28ae5bafd7e19d1c3fd7a7c65c2c681d2eaf7ffc31f5d54f016a04d41209"},"schema_version":"1.0","source":{"id":"2411.19710","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2411.19710","created_at":"2026-07-05T09:42:07Z"},{"alias_kind":"arxiv_version","alias_value":"2411.19710v1","created_at":"2026-07-05T09:42:07Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.19710","created_at":"2026-07-05T09:42:07Z"},{"alias_kind":"pith_short_12","alias_value":"NF2SXP6ISKI5","created_at":"2026-07-05T09:42:07Z"},{"alias_kind":"pith_short_16","alias_value":"NF2SXP6ISKI5XHBB","created_at":"2026-07-05T09:42:07Z"},{"alias_kind":"pith_short_8","alias_value":"NF2SXP6I","created_at":"2026-07-05T09:42:07Z"}],"graph_snapshots":[{"event_id":"sha256:7f49a67f9e3f9e4177c1f5abf658f87f0954dbdf01a610bbfdb3600541df85b8","target":"graph","created_at":"2026-07-05T09:42:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2411.19710/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Retrieval Augmented Generation (RAG) systems are a widespread application of Large Language Models (LLMs) in the industry. While many tools exist empowering developers to build their own systems, measuring their performance locally, with datasets reflective of the system's use cases, is a technological challenge. Solutions to this problem range from non-specific and cheap (most public datasets) to specific and costly (generating data from local documents). In this paper, we show that using public question and answer (Q&A) datasets to assess retrieval performance can lead to non-optimal systems","authors_text":"(2) IBM Research Zurich), Cesar Berrospi (2), Lokesh Mishra (2), Michele Dolfi (2), Panagiotis Vagenas (2) ((1) IBM Research Paris-Saclay, Peter Staar (2), Rafael Teixeira de Lima (1), Shubham Gupta (1)","cross_cats":["cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2024-11-29T13:57:07Z","title":"Know Your RAG: Dataset Taxonomy and Generation Strategies for Evaluating RAG Systems"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.19710","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:6843eb82302727b95b452b0fb6bf10e1104e9718b51f77e59d28f84154627d4f","target":"record","created_at":"2026-07-05T09:42:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"a01067a9082019d6f081d7d30278534e8b25dff2104322b7c031c9f8110242a3","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2024-11-29T13:57:07Z","title_canon_sha256":"c96b28ae5bafd7e19d1c3fd7a7c65c2c681d2eaf7ffc31f5d54f016a04d41209"},"schema_version":"1.0","source":{"id":"2411.19710","kind":"arxiv","version":1}},"canonical_sha256":"69752bbfc89291db9c21d5a97d9e0c251f75b7b077025453d4b7be4ac30088a3","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"69752bbfc89291db9c21d5a97d9e0c251f75b7b077025453d4b7be4ac30088a3","first_computed_at":"2026-07-05T09:42:07.097018Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:42:07.097018Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"H8DeK0By/GHlE9qaN4Ax6ZDiFHZMSFlr5QJxRk2dTk9AYmOqDEr0Z6LsRaMoD2AO4L4GKWHSsddP5mHY2ajfCA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:42:07.097538Z","signed_message":"canonical_sha256_bytes"},"source_id":"2411.19710","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:6843eb82302727b95b452b0fb6bf10e1104e9718b51f77e59d28f84154627d4f","sha256:7f49a67f9e3f9e4177c1f5abf658f87f0954dbdf01a610bbfdb3600541df85b8"],"state_sha256":"c74151b9952cac75eee1837e381dbc8965d7af6137561d3aeff871b4732daa1c"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"aV9qnZ37yrsm2PD95i5L/RsYna9y+hHzfT6/K6cw66xHlbgAklHbRbChaE0+KH5OyYLIPB8Ky6V9FSQ1+NLyAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-15T21:59:17.362868Z","bundle_sha256":"8bb02eef62eaf9263351c2ca703fd8f23363cfb418eb78e501ff00916cb33190"}}