{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:V4FHGP5VSRFUD7WEMIKZFLJXKB","short_pith_number":"pith:V4FHGP5V","canonical_record":{"source":{"id":"2506.13023","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-06-16T01:18:16Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"99f0b381e6a20cb5cab32a411a4df7f90e65521fb6b720ffc02888496adaf98b","abstract_canon_sha256":"829cbc3c4f1edf865d34ee50a84426c1b6b6b00f51cb33cb2d480405897c6c56"},"schema_version":"1.0"},"canonical_sha256":"af0a733fb5944b41fec4621592ad37505acad361049f17193d73a085605db01a","source":{"kind":"arxiv","id":"2506.13023","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.13023","created_at":"2026-07-05T11:39:55Z"},{"alias_kind":"arxiv_version","alias_value":"2506.13023v2","created_at":"2026-07-05T11:39:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.13023","created_at":"2026-07-05T11:39:55Z"},{"alias_kind":"pith_short_12","alias_value":"V4FHGP5VSRFU","created_at":"2026-07-05T11:39:55Z"},{"alias_kind":"pith_short_16","alias_value":"V4FHGP5VSRFUD7WE","created_at":"2026-07-05T11:39:55Z"},{"alias_kind":"pith_short_8","alias_value":"V4FHGP5V","created_at":"2026-07-05T11:39:55Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:V4FHGP5VSRFUD7WEMIKZFLJXKB","target":"record","payload":{"canonical_record":{"source":{"id":"2506.13023","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-06-16T01:18:16Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"99f0b381e6a20cb5cab32a411a4df7f90e65521fb6b720ffc02888496adaf98b","abstract_canon_sha256":"829cbc3c4f1edf865d34ee50a84426c1b6b6b00f51cb33cb2d480405897c6c56"},"schema_version":"1.0"},"canonical_sha256":"af0a733fb5944b41fec4621592ad37505acad361049f17193d73a085605db01a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:39:55.249434Z","signature_b64":"Moglg0B/umRRzozI2SectEGiyaSvXr8Wr36VIpaBa5G/5NUPiH90d+naV7AIuoSXy5WlH+W/F+Nbl4fpfcmTBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"af0a733fb5944b41fec4621592ad37505acad361049f17193d73a085605db01a","last_reissued_at":"2026-07-05T11:39:55.248958Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:39:55.248958Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2506.13023","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:39:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"wFqfXXJL39dmXUe7zSYmop1GEnj5fHAFWWu4V+E2yEJdt8f3p1nI9y7bmCeGX9WYA8qQlYJlh+q5P0Yg/dTnCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T04:00:59.618778Z"},"content_sha256":"572c3113ce497009a072584b7b47f9aba4aaa2b9e1546cbe514f5d09700c7edd","schema_version":"1.0","event_id":"sha256:572c3113ce497009a072584b7b47f9aba4aaa2b9e1546cbe514f5d09700c7edd"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:V4FHGP5VSRFUD7WEMIKZFLJXKB","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"A Practical Guide for Evaluating LLMs and LLM-Reliant Systems","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.AI","authors_text":"Christopher Andrews, Ethan M. Rudd, Philip Tully","submitted_at":"2025-06-16T01:18:16Z","abstract_excerpt":"Recent advances in generative AI have led to remarkable interest in using systems that rely on large language models (LLMs) for practical applications. However, meaningful evaluation of these systems in real-world scenarios comes with a distinct set of challenges, which are not well-addressed by synthetic benchmarks and de-facto metrics that are often seen in the literature. We present a practical evaluation framework which outlines how to proactively curate representative datasets, select meaningful evaluation metrics, and employ meaningful evaluation methodologies that integrate well with pr"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.13023","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.13023/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:39:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"VZqeEwxX8v5PBTmy+P/iQdpao5TyFXgWyvqoQrpbC/HLsEZKzMTsUxprZuKhYKUgcNw9xLX10BW/QWLPgT6fAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T04:00:59.619277Z"},"content_sha256":"2c836ef77dad3eef71d678f767b0ca5fa30227519afac7fd37589e996c928c8f","schema_version":"1.0","event_id":"sha256:2c836ef77dad3eef71d678f767b0ca5fa30227519afac7fd37589e996c928c8f"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/V4FHGP5VSRFUD7WEMIKZFLJXKB/bundle.json","state_url":"https://pith.science/pith/V4FHGP5VSRFUD7WEMIKZFLJXKB/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/V4FHGP5VSRFUD7WEMIKZFLJXKB/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T04:00:59Z","links":{"resolver":"https://pith.science/pith/V4FHGP5VSRFUD7WEMIKZFLJXKB","bundle":"https://pith.science/pith/V4FHGP5VSRFUD7WEMIKZFLJXKB/bundle.json","state":"https://pith.science/pith/V4FHGP5VSRFUD7WEMIKZFLJXKB/state.json","well_known_bundle":"https://pith.science/.well-known/pith/V4FHGP5VSRFUD7WEMIKZFLJXKB/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:V4FHGP5VSRFUD7WEMIKZFLJXKB","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"829cbc3c4f1edf865d34ee50a84426c1b6b6b00f51cb33cb2d480405897c6c56","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-06-16T01:18:16Z","title_canon_sha256":"99f0b381e6a20cb5cab32a411a4df7f90e65521fb6b720ffc02888496adaf98b"},"schema_version":"1.0","source":{"id":"2506.13023","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.13023","created_at":"2026-07-05T11:39:55Z"},{"alias_kind":"arxiv_version","alias_value":"2506.13023v2","created_at":"2026-07-05T11:39:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.13023","created_at":"2026-07-05T11:39:55Z"},{"alias_kind":"pith_short_12","alias_value":"V4FHGP5VSRFU","created_at":"2026-07-05T11:39:55Z"},{"alias_kind":"pith_short_16","alias_value":"V4FHGP5VSRFUD7WE","created_at":"2026-07-05T11:39:55Z"},{"alias_kind":"pith_short_8","alias_value":"V4FHGP5V","created_at":"2026-07-05T11:39:55Z"}],"graph_snapshots":[{"event_id":"sha256:2c836ef77dad3eef71d678f767b0ca5fa30227519afac7fd37589e996c928c8f","target":"graph","created_at":"2026-07-05T11:39:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2506.13023/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent advances in generative AI have led to remarkable interest in using systems that rely on large language models (LLMs) for practical applications. However, meaningful evaluation of these systems in real-world scenarios comes with a distinct set of challenges, which are not well-addressed by synthetic benchmarks and de-facto metrics that are often seen in the literature. We present a practical evaluation framework which outlines how to proactively curate representative datasets, select meaningful evaluation metrics, and employ meaningful evaluation methodologies that integrate well with pr","authors_text":"Christopher Andrews, Ethan M. Rudd, Philip Tully","cross_cats":["cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-06-16T01:18:16Z","title":"A Practical Guide for Evaluating LLMs and LLM-Reliant Systems"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.13023","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:572c3113ce497009a072584b7b47f9aba4aaa2b9e1546cbe514f5d09700c7edd","target":"record","created_at":"2026-07-05T11:39:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"829cbc3c4f1edf865d34ee50a84426c1b6b6b00f51cb33cb2d480405897c6c56","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-06-16T01:18:16Z","title_canon_sha256":"99f0b381e6a20cb5cab32a411a4df7f90e65521fb6b720ffc02888496adaf98b"},"schema_version":"1.0","source":{"id":"2506.13023","kind":"arxiv","version":2}},"canonical_sha256":"af0a733fb5944b41fec4621592ad37505acad361049f17193d73a085605db01a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"af0a733fb5944b41fec4621592ad37505acad361049f17193d73a085605db01a","first_computed_at":"2026-07-05T11:39:55.248958Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:39:55.248958Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Moglg0B/umRRzozI2SectEGiyaSvXr8Wr36VIpaBa5G/5NUPiH90d+naV7AIuoSXy5WlH+W/F+Nbl4fpfcmTBA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:39:55.249434Z","signed_message":"canonical_sha256_bytes"},"source_id":"2506.13023","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:572c3113ce497009a072584b7b47f9aba4aaa2b9e1546cbe514f5d09700c7edd","sha256:2c836ef77dad3eef71d678f767b0ca5fa30227519afac7fd37589e996c928c8f"],"state_sha256":"889d743aa7acafca4abd0eac2a8e63680d1320f2ad4e830af8a893a3ca05c38e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"95fyBo9O4cDIm1ws1EG5DW8B9B8R0K1vj3rL/u13jeUllxjFWmd9cBVEPbJUiKAjvNnoXCdEIyqnO9ymjC5lAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T04:00:59.623036Z","bundle_sha256":"1f08b08865097ed1f0fd54ebbcd53f7e80dac4ff9864a3d7d3f731ac6872b088"}}