{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:VN4LIJQZ7VMAOMFDM27OZUZUQU","short_pith_number":"pith:VN4LIJQZ","canonical_record":{"source":{"id":"2503.05336","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-03-07T11:23:48Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"1ae84b53b06e84a01e6b2753c0b26066c612e8488fe63aee6b3b90645cf71b2a","abstract_canon_sha256":"c9418243a78c0ce8ba1ed267235f999494dd0e17c1439d4eeba392f88152cd5e"},"schema_version":"1.0"},"canonical_sha256":"ab78b42619fd580730a366beecd33485064a030052dd970fe3c1e8c01120ea3f","source":{"kind":"arxiv","id":"2503.05336","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.05336","created_at":"2026-07-05T10:30:32Z"},{"alias_kind":"arxiv_version","alias_value":"2503.05336v3","created_at":"2026-07-05T10:30:32Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.05336","created_at":"2026-07-05T10:30:32Z"},{"alias_kind":"pith_short_12","alias_value":"VN4LIJQZ7VMA","created_at":"2026-07-05T10:30:32Z"},{"alias_kind":"pith_short_16","alias_value":"VN4LIJQZ7VMAOMFD","created_at":"2026-07-05T10:30:32Z"},{"alias_kind":"pith_short_8","alias_value":"VN4LIJQZ","created_at":"2026-07-05T10:30:32Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:VN4LIJQZ7VMAOMFDM27OZUZUQU","target":"record","payload":{"canonical_record":{"source":{"id":"2503.05336","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-03-07T11:23:48Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"1ae84b53b06e84a01e6b2753c0b26066c612e8488fe63aee6b3b90645cf71b2a","abstract_canon_sha256":"c9418243a78c0ce8ba1ed267235f999494dd0e17c1439d4eeba392f88152cd5e"},"schema_version":"1.0"},"canonical_sha256":"ab78b42619fd580730a366beecd33485064a030052dd970fe3c1e8c01120ea3f","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:30:32.463031Z","signature_b64":"p/TlcjKV8hoMxN1m/jcHTIZW1UnqovyFEpK3F0PQr53XzS1qi7B6Ptyxmy3Ir0b0lohlxD4BgFni+qGdWcb8CQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ab78b42619fd580730a366beecd33485064a030052dd970fe3c1e8c01120ea3f","last_reissued_at":"2026-07-05T10:30:32.462301Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:30:32.462301Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2503.05336","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:30:32Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"luccJOSl5DG0BB4wfrkBJvDLhnCsNcuN7QxEzFEZQ6DH+MGPaTUqQuRcBpirHOCoEbUEmti/R71s+F8f4DOeCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-15T00:02:52.614943Z"},"content_sha256":"b7f01ae35c742ad64bc1a2c1b030d0b2841d01adfd0c2f442fbc5b6d1cda29da","schema_version":"1.0","event_id":"sha256:b7f01ae35c742ad64bc1a2c1b030d0b2841d01adfd0c2f442fbc5b6d1cda29da"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:VN4LIJQZ7VMAOMFDM27OZUZUQU","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Toward an Evaluation Science for Generative AI Systems","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.AI","authors_text":"Angelina Wang, Deep Ganguli, Hanna Wallach, Inioluwa Deborah Raji, Laura Weidinger, Margaret Mitchell, Olawale Salaudeen, Rishi Bommasani, Sanmi Koyejo, William Isaac","submitted_at":"2025-03-07T11:23:48Z","abstract_excerpt":"There is an increasing imperative to anticipate and understand the performance and safety of generative AI systems in real-world deployment contexts. However, the current evaluation ecosystem is insufficient: Commonly used static benchmarks face validity challenges, and ad hoc case-by-case audits rarely scale. In this piece, we advocate for maturing an evaluation science for generative AI systems. While generative AI creates unique challenges for system safety engineering and measurement science, the field can draw valuable insights from the development of safety evaluation practices in other "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.05336","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2503.05336/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:30:32Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"/qS2EYq4Wc00DP3G6cJoczdz3BEWtcBrTeaaUS4Q41eKApxpT299eC1y6dLHF8OIkbboYJM7o5K8OFwfq1hVCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-15T00:02:52.615482Z"},"content_sha256":"37af03804da6475fd22f51a8972b52c7520ea828408a6b44e1e3c72c4c60b785","schema_version":"1.0","event_id":"sha256:37af03804da6475fd22f51a8972b52c7520ea828408a6b44e1e3c72c4c60b785"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/VN4LIJQZ7VMAOMFDM27OZUZUQU/bundle.json","state_url":"https://pith.science/pith/VN4LIJQZ7VMAOMFDM27OZUZUQU/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/VN4LIJQZ7VMAOMFDM27OZUZUQU/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-15T00:02:52Z","links":{"resolver":"https://pith.science/pith/VN4LIJQZ7VMAOMFDM27OZUZUQU","bundle":"https://pith.science/pith/VN4LIJQZ7VMAOMFDM27OZUZUQU/bundle.json","state":"https://pith.science/pith/VN4LIJQZ7VMAOMFDM27OZUZUQU/state.json","well_known_bundle":"https://pith.science/.well-known/pith/VN4LIJQZ7VMAOMFDM27OZUZUQU/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:VN4LIJQZ7VMAOMFDM27OZUZUQU","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c9418243a78c0ce8ba1ed267235f999494dd0e17c1439d4eeba392f88152cd5e","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-03-07T11:23:48Z","title_canon_sha256":"1ae84b53b06e84a01e6b2753c0b26066c612e8488fe63aee6b3b90645cf71b2a"},"schema_version":"1.0","source":{"id":"2503.05336","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.05336","created_at":"2026-07-05T10:30:32Z"},{"alias_kind":"arxiv_version","alias_value":"2503.05336v3","created_at":"2026-07-05T10:30:32Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.05336","created_at":"2026-07-05T10:30:32Z"},{"alias_kind":"pith_short_12","alias_value":"VN4LIJQZ7VMA","created_at":"2026-07-05T10:30:32Z"},{"alias_kind":"pith_short_16","alias_value":"VN4LIJQZ7VMAOMFD","created_at":"2026-07-05T10:30:32Z"},{"alias_kind":"pith_short_8","alias_value":"VN4LIJQZ","created_at":"2026-07-05T10:30:32Z"}],"graph_snapshots":[{"event_id":"sha256:37af03804da6475fd22f51a8972b52c7520ea828408a6b44e1e3c72c4c60b785","target":"graph","created_at":"2026-07-05T10:30:32Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2503.05336/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"There is an increasing imperative to anticipate and understand the performance and safety of generative AI systems in real-world deployment contexts. However, the current evaluation ecosystem is insufficient: Commonly used static benchmarks face validity challenges, and ad hoc case-by-case audits rarely scale. In this piece, we advocate for maturing an evaluation science for generative AI systems. While generative AI creates unique challenges for system safety engineering and measurement science, the field can draw valuable insights from the development of safety evaluation practices in other ","authors_text":"Angelina Wang, Deep Ganguli, Hanna Wallach, Inioluwa Deborah Raji, Laura Weidinger, Margaret Mitchell, Olawale Salaudeen, Rishi Bommasani, Sanmi Koyejo, William Isaac","cross_cats":["cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-03-07T11:23:48Z","title":"Toward an Evaluation Science for Generative AI Systems"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.05336","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b7f01ae35c742ad64bc1a2c1b030d0b2841d01adfd0c2f442fbc5b6d1cda29da","target":"record","created_at":"2026-07-05T10:30:32Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c9418243a78c0ce8ba1ed267235f999494dd0e17c1439d4eeba392f88152cd5e","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-03-07T11:23:48Z","title_canon_sha256":"1ae84b53b06e84a01e6b2753c0b26066c612e8488fe63aee6b3b90645cf71b2a"},"schema_version":"1.0","source":{"id":"2503.05336","kind":"arxiv","version":3}},"canonical_sha256":"ab78b42619fd580730a366beecd33485064a030052dd970fe3c1e8c01120ea3f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"ab78b42619fd580730a366beecd33485064a030052dd970fe3c1e8c01120ea3f","first_computed_at":"2026-07-05T10:30:32.462301Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:30:32.462301Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"p/TlcjKV8hoMxN1m/jcHTIZW1UnqovyFEpK3F0PQr53XzS1qi7B6Ptyxmy3Ir0b0lohlxD4BgFni+qGdWcb8CQ==","signature_status":"signed_v1","signed_at":"2026-07-05T10:30:32.463031Z","signed_message":"canonical_sha256_bytes"},"source_id":"2503.05336","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b7f01ae35c742ad64bc1a2c1b030d0b2841d01adfd0c2f442fbc5b6d1cda29da","sha256:37af03804da6475fd22f51a8972b52c7520ea828408a6b44e1e3c72c4c60b785"],"state_sha256":"58cf350577dd39704a5a20867d0209212f385b1f6a48c57433542fcb935fa39f"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"vzuGy7UitkTCYYfdU3XhgF6o5OXqa4oWLAiTg+3R6Y4bVcgDZXXEIVR9e7KPP9Oga/IantijDslLtApzM6D3DA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-15T00:02:52.621766Z","bundle_sha256":"8a9ed85016093416d161148398b596c32fac21f2c3843ec11398d2c444c7159a"}}