{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:EX274QEHZ5G3GEQVWBOHLCI2WI","short_pith_number":"pith:EX274QEH","canonical_record":{"source":{"id":"2505.00612","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-05-01T15:43:51Z","cross_cats_sorted":[],"title_canon_sha256":"0fcdb3575f877e4e91ca26e663394cefb0df3d2f7f547d20b298371aa305a385","abstract_canon_sha256":"26dd467b4a15d818865ecdc70e9b16f0779136c2605f59eab72c65661c2965e5"},"schema_version":"1.0"},"canonical_sha256":"25f5fe4087cf4db31215b05c75891ab2034de06d1828d3c0958838689e402cef","source":{"kind":"arxiv","id":"2505.00612","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.00612","created_at":"2026-07-05T11:11:44Z"},{"alias_kind":"arxiv_version","alias_value":"2505.00612v2","created_at":"2026-07-05T11:11:44Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.00612","created_at":"2026-07-05T11:11:44Z"},{"alias_kind":"pith_short_12","alias_value":"EX274QEHZ5G3","created_at":"2026-07-05T11:11:44Z"},{"alias_kind":"pith_short_16","alias_value":"EX274QEHZ5G3GEQV","created_at":"2026-07-05T11:11:44Z"},{"alias_kind":"pith_short_8","alias_value":"EX274QEH","created_at":"2026-07-05T11:11:44Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:EX274QEHZ5G3GEQVWBOHLCI2WI","target":"record","payload":{"canonical_record":{"source":{"id":"2505.00612","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-05-01T15:43:51Z","cross_cats_sorted":[],"title_canon_sha256":"0fcdb3575f877e4e91ca26e663394cefb0df3d2f7f547d20b298371aa305a385","abstract_canon_sha256":"26dd467b4a15d818865ecdc70e9b16f0779136c2605f59eab72c65661c2965e5"},"schema_version":"1.0"},"canonical_sha256":"25f5fe4087cf4db31215b05c75891ab2034de06d1828d3c0958838689e402cef","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:11:44.441856Z","signature_b64":"lJa+SQ+cQUu6A4liSE4S51JxvMj3ys/VqaOqrPtlJmHXMNqBD2uWn7KxVDXrn5Op3PbxyT5Eqb+GeRK0gZZABA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"25f5fe4087cf4db31215b05c75891ab2034de06d1828d3c0958838689e402cef","last_reissued_at":"2026-07-05T11:11:44.441400Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:11:44.441400Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2505.00612","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:11:44Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"IMzSKS63qpkqclLYGIE+HMesuW1mGArEhtbRWt0biD2v5Gu9ZnlFSPokAQufirAB3MDxyzQrLidI1+tztE9nAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T22:02:43.657409Z"},"content_sha256":"5ba8433884483e24670ca6e6b1fea6a22a1e9c41f1261597d1864e2f8abd52ee","schema_version":"1.0","event_id":"sha256:5ba8433884483e24670ca6e6b1fea6a22a1e9c41f1261597d1864e2f8abd52ee"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:EX274QEHZ5G3GEQVWBOHLCI2WI","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Position: AI Competitions Provide the Gold Standard for Empirical Rigor in GenAI Evaluation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Addison Howard, D. Sculley, Maggie Demkin, Megan Risdal, Nate Keating, Paul Mooney, Phil Culliton, Ryan Holbrook, Sohier Dane, Walter Reade, Will Cukierski","submitted_at":"2025-05-01T15:43:51Z","abstract_excerpt":"In this position paper, we observe that empirical evaluation in Generative AI is at a crisis point since traditional ML evaluation and benchmarking strategies are insufficient to meet the needs of evaluating modern GenAI models and systems. There are many reasons for this, including the fact that these models typically have nearly unbounded input and output spaces, typically do not have a well defined ground truth target, and typically exhibit strong feedback loops and prediction dependence based on context of previous model outputs. On top of these critical issues, we argue that the problems "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.00612","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2505.00612/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:11:44Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"9Lf9+7R6ApAurRrdw3642tqqvUu4IDD+sLZwdz6N/uJEP8UqyCShi45CAFvkUY/6XNQ24EA5ku4KTBZWOqNQBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T22:02:43.657922Z"},"content_sha256":"be2c7e20de895207a4de1cfa50e215f7ac69f2c1a1f0de3a3c91eae32e4fce12","schema_version":"1.0","event_id":"sha256:be2c7e20de895207a4de1cfa50e215f7ac69f2c1a1f0de3a3c91eae32e4fce12"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/EX274QEHZ5G3GEQVWBOHLCI2WI/bundle.json","state_url":"https://pith.science/pith/EX274QEHZ5G3GEQVWBOHLCI2WI/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/EX274QEHZ5G3GEQVWBOHLCI2WI/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T22:02:43Z","links":{"resolver":"https://pith.science/pith/EX274QEHZ5G3GEQVWBOHLCI2WI","bundle":"https://pith.science/pith/EX274QEHZ5G3GEQVWBOHLCI2WI/bundle.json","state":"https://pith.science/pith/EX274QEHZ5G3GEQVWBOHLCI2WI/state.json","well_known_bundle":"https://pith.science/.well-known/pith/EX274QEHZ5G3GEQVWBOHLCI2WI/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:EX274QEHZ5G3GEQVWBOHLCI2WI","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"26dd467b4a15d818865ecdc70e9b16f0779136c2605f59eab72c65661c2965e5","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-05-01T15:43:51Z","title_canon_sha256":"0fcdb3575f877e4e91ca26e663394cefb0df3d2f7f547d20b298371aa305a385"},"schema_version":"1.0","source":{"id":"2505.00612","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.00612","created_at":"2026-07-05T11:11:44Z"},{"alias_kind":"arxiv_version","alias_value":"2505.00612v2","created_at":"2026-07-05T11:11:44Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.00612","created_at":"2026-07-05T11:11:44Z"},{"alias_kind":"pith_short_12","alias_value":"EX274QEHZ5G3","created_at":"2026-07-05T11:11:44Z"},{"alias_kind":"pith_short_16","alias_value":"EX274QEHZ5G3GEQV","created_at":"2026-07-05T11:11:44Z"},{"alias_kind":"pith_short_8","alias_value":"EX274QEH","created_at":"2026-07-05T11:11:44Z"}],"graph_snapshots":[{"event_id":"sha256:be2c7e20de895207a4de1cfa50e215f7ac69f2c1a1f0de3a3c91eae32e4fce12","target":"graph","created_at":"2026-07-05T11:11:44Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2505.00612/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In this position paper, we observe that empirical evaluation in Generative AI is at a crisis point since traditional ML evaluation and benchmarking strategies are insufficient to meet the needs of evaluating modern GenAI models and systems. There are many reasons for this, including the fact that these models typically have nearly unbounded input and output spaces, typically do not have a well defined ground truth target, and typically exhibit strong feedback loops and prediction dependence based on context of previous model outputs. On top of these critical issues, we argue that the problems ","authors_text":"Addison Howard, D. Sculley, Maggie Demkin, Megan Risdal, Nate Keating, Paul Mooney, Phil Culliton, Ryan Holbrook, Sohier Dane, Walter Reade, Will Cukierski","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-05-01T15:43:51Z","title":"Position: AI Competitions Provide the Gold Standard for Empirical Rigor in GenAI Evaluation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.00612","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5ba8433884483e24670ca6e6b1fea6a22a1e9c41f1261597d1864e2f8abd52ee","target":"record","created_at":"2026-07-05T11:11:44Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"26dd467b4a15d818865ecdc70e9b16f0779136c2605f59eab72c65661c2965e5","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-05-01T15:43:51Z","title_canon_sha256":"0fcdb3575f877e4e91ca26e663394cefb0df3d2f7f547d20b298371aa305a385"},"schema_version":"1.0","source":{"id":"2505.00612","kind":"arxiv","version":2}},"canonical_sha256":"25f5fe4087cf4db31215b05c75891ab2034de06d1828d3c0958838689e402cef","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"25f5fe4087cf4db31215b05c75891ab2034de06d1828d3c0958838689e402cef","first_computed_at":"2026-07-05T11:11:44.441400Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:11:44.441400Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"lJa+SQ+cQUu6A4liSE4S51JxvMj3ys/VqaOqrPtlJmHXMNqBD2uWn7KxVDXrn5Op3PbxyT5Eqb+GeRK0gZZABA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:11:44.441856Z","signed_message":"canonical_sha256_bytes"},"source_id":"2505.00612","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5ba8433884483e24670ca6e6b1fea6a22a1e9c41f1261597d1864e2f8abd52ee","sha256:be2c7e20de895207a4de1cfa50e215f7ac69f2c1a1f0de3a3c91eae32e4fce12"],"state_sha256":"13d24f83a66cf081e40c0e49571854b77aafe1676f6fde72f3c07986f6415d18"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ds8/7HjZM5V9Km09MEFBnFRR7PUuzinyvtMj2Nhr9IqQo8TTkZbNC+aFXhlDu4QhKz3t5Ot7ddcsNv69JJWACA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T22:02:43.662331Z","bundle_sha256":"81c732a2ca509be75424e5849cc9032e33caa40c7c3bf824002398cde59620f1"}}