{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:ATPEOXGBXWXRQYB3JELL2LCR2C","short_pith_number":"pith:ATPEOXGB","canonical_record":{"source":{"id":"2504.16778","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-04-23T14:52:39Z","cross_cats_sorted":["cs.AI","cs.CY"],"title_canon_sha256":"a95926e7adbfca33b71381a2be14fcc3856a2d3c3cf062b586e0c98a83ae36ca","abstract_canon_sha256":"297b0ea990c13b48cc37e86b393c4e21d25c97f660474759ab0507105173b1bd"},"schema_version":"1.0"},"canonical_sha256":"04de475cc1bdaf18603b4916bd2c51d086049b3efe1765789d8a988cfd213ef1","source":{"kind":"arxiv","id":"2504.16778","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2504.16778","created_at":"2026-07-05T10:54:47Z"},{"alias_kind":"arxiv_version","alias_value":"2504.16778v2","created_at":"2026-07-05T10:54:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.16778","created_at":"2026-07-05T10:54:47Z"},{"alias_kind":"pith_short_12","alias_value":"ATPEOXGBXWXR","created_at":"2026-07-05T10:54:47Z"},{"alias_kind":"pith_short_16","alias_value":"ATPEOXGBXWXRQYB3","created_at":"2026-07-05T10:54:47Z"},{"alias_kind":"pith_short_8","alias_value":"ATPEOXGB","created_at":"2026-07-05T10:54:47Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:ATPEOXGBXWXRQYB3JELL2LCR2C","target":"record","payload":{"canonical_record":{"source":{"id":"2504.16778","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-04-23T14:52:39Z","cross_cats_sorted":["cs.AI","cs.CY"],"title_canon_sha256":"a95926e7adbfca33b71381a2be14fcc3856a2d3c3cf062b586e0c98a83ae36ca","abstract_canon_sha256":"297b0ea990c13b48cc37e86b393c4e21d25c97f660474759ab0507105173b1bd"},"schema_version":"1.0"},"canonical_sha256":"04de475cc1bdaf18603b4916bd2c51d086049b3efe1765789d8a988cfd213ef1","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:54:47.823499Z","signature_b64":"n9aJqg1vQhjq2hxpWy/4Hs9tzlfcWgB6c7x0tijWPkz9EEJP5xvQQvDo14cjZEL3etiVBAaXTCY8B+hLcHszCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"04de475cc1bdaf18603b4916bd2c51d086049b3efe1765789d8a988cfd213ef1","last_reissued_at":"2026-07-05T10:54:47.823018Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:54:47.823018Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2504.16778","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:54:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"rYMe688YvgIDVihm/yfCzEta02tAa4C8xOo55hlMmtebI9dexvifOU96ebQE0Dcav1LIwWX22gKVOPybYJ+HAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-22T18:47:20.820560Z"},"content_sha256":"c96b6ffe563461bff70c7b07cd6a42f23b006ad880b2b4ce918fa3f403f2aa2d","schema_version":"1.0","event_id":"sha256:c96b6ffe563461bff70c7b07cd6a42f23b006ad880b2b4ce918fa3f403f2aa2d"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:ATPEOXGBXWXRQYB3JELL2LCR2C","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Evaluation Framework for AI Systems in \"the Wild\"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CY"],"primary_cat":"cs.CL","authors_text":"Anindya Das Antar, Bryan Goodman, David Jurgens, Elizabeth Bondi-Kelly, Insu Jang, Jae-Won Chung, Jiachen Liu, Joseph Peper, Kevin Samy, Lu Wang, Michael Wellman, Mosharaf Chowdhury, Rada Mihalcea, Sarah Jabbour, Shiqi He, Trenton Chang","submitted_at":"2025-04-23T14:52:39Z","abstract_excerpt":"Generative AI (GenAI) models have become vital across industries, yet current evaluation methods have not adapted to their widespread use. Traditional evaluations often rely on benchmarks and fixed datasets, frequently failing to reflect real-world performance, which creates a gap between lab-tested outcomes and practical applications. This white paper proposes a comprehensive framework for how we should evaluate real-world GenAI systems, emphasizing diverse, evolving inputs and holistic, dynamic, and ongoing assessment approaches. The paper offers guidance for practitioners on how to design e"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2504.16778","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2504.16778/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:54:47Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jTB6bal9P8ZqPx+W0lkLCZ4Tvfl6XF+labzyB1F0s4dTTsedOu0xpX+n5441aB6vvXmrxL8S1FN9SFGwXY9oAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-22T18:47:20.821557Z"},"content_sha256":"ec16e1b25e4746e7002f50fc7385a93343aaeeae23f4fcb934e4a69d922eecf1","schema_version":"1.0","event_id":"sha256:ec16e1b25e4746e7002f50fc7385a93343aaeeae23f4fcb934e4a69d922eecf1"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ATPEOXGBXWXRQYB3JELL2LCR2C/bundle.json","state_url":"https://pith.science/pith/ATPEOXGBXWXRQYB3JELL2LCR2C/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ATPEOXGBXWXRQYB3JELL2LCR2C/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-22T18:47:20Z","links":{"resolver":"https://pith.science/pith/ATPEOXGBXWXRQYB3JELL2LCR2C","bundle":"https://pith.science/pith/ATPEOXGBXWXRQYB3JELL2LCR2C/bundle.json","state":"https://pith.science/pith/ATPEOXGBXWXRQYB3JELL2LCR2C/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ATPEOXGBXWXRQYB3JELL2LCR2C/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:ATPEOXGBXWXRQYB3JELL2LCR2C","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"297b0ea990c13b48cc37e86b393c4e21d25c97f660474759ab0507105173b1bd","cross_cats_sorted":["cs.AI","cs.CY"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-04-23T14:52:39Z","title_canon_sha256":"a95926e7adbfca33b71381a2be14fcc3856a2d3c3cf062b586e0c98a83ae36ca"},"schema_version":"1.0","source":{"id":"2504.16778","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2504.16778","created_at":"2026-07-05T10:54:47Z"},{"alias_kind":"arxiv_version","alias_value":"2504.16778v2","created_at":"2026-07-05T10:54:47Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.16778","created_at":"2026-07-05T10:54:47Z"},{"alias_kind":"pith_short_12","alias_value":"ATPEOXGBXWXR","created_at":"2026-07-05T10:54:47Z"},{"alias_kind":"pith_short_16","alias_value":"ATPEOXGBXWXRQYB3","created_at":"2026-07-05T10:54:47Z"},{"alias_kind":"pith_short_8","alias_value":"ATPEOXGB","created_at":"2026-07-05T10:54:47Z"}],"graph_snapshots":[{"event_id":"sha256:ec16e1b25e4746e7002f50fc7385a93343aaeeae23f4fcb934e4a69d922eecf1","target":"graph","created_at":"2026-07-05T10:54:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2504.16778/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Generative AI (GenAI) models have become vital across industries, yet current evaluation methods have not adapted to their widespread use. Traditional evaluations often rely on benchmarks and fixed datasets, frequently failing to reflect real-world performance, which creates a gap between lab-tested outcomes and practical applications. This white paper proposes a comprehensive framework for how we should evaluate real-world GenAI systems, emphasizing diverse, evolving inputs and holistic, dynamic, and ongoing assessment approaches. The paper offers guidance for practitioners on how to design e","authors_text":"Anindya Das Antar, Bryan Goodman, David Jurgens, Elizabeth Bondi-Kelly, Insu Jang, Jae-Won Chung, Jiachen Liu, Joseph Peper, Kevin Samy, Lu Wang, Michael Wellman, Mosharaf Chowdhury, Rada Mihalcea, Sarah Jabbour, Shiqi He, Trenton Chang","cross_cats":["cs.AI","cs.CY"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-04-23T14:52:39Z","title":"Evaluation Framework for AI Systems in \"the Wild\""},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2504.16778","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:c96b6ffe563461bff70c7b07cd6a42f23b006ad880b2b4ce918fa3f403f2aa2d","target":"record","created_at":"2026-07-05T10:54:47Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"297b0ea990c13b48cc37e86b393c4e21d25c97f660474759ab0507105173b1bd","cross_cats_sorted":["cs.AI","cs.CY"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-04-23T14:52:39Z","title_canon_sha256":"a95926e7adbfca33b71381a2be14fcc3856a2d3c3cf062b586e0c98a83ae36ca"},"schema_version":"1.0","source":{"id":"2504.16778","kind":"arxiv","version":2}},"canonical_sha256":"04de475cc1bdaf18603b4916bd2c51d086049b3efe1765789d8a988cfd213ef1","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"04de475cc1bdaf18603b4916bd2c51d086049b3efe1765789d8a988cfd213ef1","first_computed_at":"2026-07-05T10:54:47.823018Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:54:47.823018Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"n9aJqg1vQhjq2hxpWy/4Hs9tzlfcWgB6c7x0tijWPkz9EEJP5xvQQvDo14cjZEL3etiVBAaXTCY8B+hLcHszCQ==","signature_status":"signed_v1","signed_at":"2026-07-05T10:54:47.823499Z","signed_message":"canonical_sha256_bytes"},"source_id":"2504.16778","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:c96b6ffe563461bff70c7b07cd6a42f23b006ad880b2b4ce918fa3f403f2aa2d","sha256:ec16e1b25e4746e7002f50fc7385a93343aaeeae23f4fcb934e4a69d922eecf1"],"state_sha256":"aa5ba6a05587bc1e1e401b59b6d87775a19115d409b4f12022b1aba452f46a26"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"YyHRXIO4+DgmTHHeRi0xa9P+utA5lp8pWu5XsNxP5xJGUZYphFNY9pXkWLcGdrz6fr0q+I/dX/mVm7+I54+MCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-22T18:47:20.827242Z","bundle_sha256":"0b5c3b4a4a02fb4b7279b71e8f4f6ff1c31ed74fff4a3a77db473b50823928f8"}}