{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2026:QIAYVDZQMYBWLTOL4RQNEUEWRW","short_pith_number":"pith:QIAYVDZQ","schema_version":"1.0","canonical_sha256":"82018a8f30660365cdcbe460d250968daf955b61714ae1bcf3207b61bb263f49","source":{"kind":"arxiv","id":"2605.20520","version":1},"attestation_state":"computed","paper":{"title":"Open-World Evaluations for Measuring Frontier AI Capabilities","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Andrew B. Hall, Andrew Schwartz, Arvind Narayanan, Cozmin Ududec, Dimitris Papailiopoulos, Gillian K Hadfield, Harry Coppock, Helen Toner, J.J. Allaire, Magda Dubois, Peter Kirgis, Rishi Bommasani, Sara Hooker, Sayash Kapoor, Seth Lazar, Shoshannah Tekofsky, Stephan Rabanser, Steve Newman","submitted_at":"2026-05-19T21:42:32Z","abstract_excerpt":"Benchmark-based evaluation remains important for tracking frontier AI progress. But it can both overstate and understate deployed capability because it privileges tasks that can be precisely specified, automatically graded, easy to optimize for, and run with low budgets and short time horizons. We advocate for a complementary class of evaluations, which we term open-world evaluations: long-horizon, messy, real-world tasks assessed through small-sample qualitative analysis rather than benchmark-scale automation. In this paper we survey recent open-world evaluations, identify their strengths and"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2605.20520","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2026-05-19T21:42:32Z","cross_cats_sorted":[],"title_canon_sha256":"3e2bb5b85f5e86ddd7423f38c6ed5c7f7b46d2a62840c2bd0b150c9c6181cdb3","abstract_canon_sha256":"12d078cc3652f496c36b2ce6f963599fe8e583753f8a6a9c0c87492c9476387a"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-21T01:04:40.814704Z","signature_b64":"CYOe8wBkvG2fHOLYwnFjxjGjWKu2hm5U95B42BLN1C9TMnkF75Vz6mNPMFGnWUahW2mbP78hMz2eVLDVmAhVAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"82018a8f30660365cdcbe460d250968daf955b61714ae1bcf3207b61bb263f49","last_reissued_at":"2026-05-21T01:04:40.814265Z","signature_status":"signed_v1","first_computed_at":"2026-05-21T01:04:40.814265Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Open-World Evaluations for Measuring Frontier AI Capabilities","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Andrew B. Hall, Andrew Schwartz, Arvind Narayanan, Cozmin Ududec, Dimitris Papailiopoulos, Gillian K Hadfield, Harry Coppock, Helen Toner, J.J. Allaire, Magda Dubois, Peter Kirgis, Rishi Bommasani, Sara Hooker, Sayash Kapoor, Seth Lazar, Shoshannah Tekofsky, Stephan Rabanser, Steve Newman","submitted_at":"2026-05-19T21:42:32Z","abstract_excerpt":"Benchmark-based evaluation remains important for tracking frontier AI progress. But it can both overstate and understate deployed capability because it privileges tasks that can be precisely specified, automatically graded, easy to optimize for, and run with low budgets and short time horizons. We advocate for a complementary class of evaluations, which we term open-world evaluations: long-horizon, messy, real-world tasks assessed through small-sample qualitative analysis rather than benchmark-scale automation. In this paper we survey recent open-world evaluations, identify their strengths and"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2605.20520","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2605.20520/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2605.20520","created_at":"2026-05-21T01:04:40.814332+00:00"},{"alias_kind":"arxiv_version","alias_value":"2605.20520v1","created_at":"2026-05-21T01:04:40.814332+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2605.20520","created_at":"2026-05-21T01:04:40.814332+00:00"},{"alias_kind":"pith_short_12","alias_value":"QIAYVDZQMYBW","created_at":"2026-05-21T01:04:40.814332+00:00"},{"alias_kind":"pith_short_16","alias_value":"QIAYVDZQMYBWLTOL","created_at":"2026-05-21T01:04:40.814332+00:00"},{"alias_kind":"pith_short_8","alias_value":"QIAYVDZQ","created_at":"2026-05-21T01:04:40.814332+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/QIAYVDZQMYBWLTOL4RQNEUEWRW","json":"https://pith.science/pith/QIAYVDZQMYBWLTOL4RQNEUEWRW.json","graph_json":"https://pith.science/api/pith-number/QIAYVDZQMYBWLTOL4RQNEUEWRW/graph.json","events_json":"https://pith.science/api/pith-number/QIAYVDZQMYBWLTOL4RQNEUEWRW/events.json","paper":"https://pith.science/paper/QIAYVDZQ"},"agent_actions":{"view_html":"https://pith.science/pith/QIAYVDZQMYBWLTOL4RQNEUEWRW","download_json":"https://pith.science/pith/QIAYVDZQMYBWLTOL4RQNEUEWRW.json","view_paper":"https://pith.science/paper/QIAYVDZQ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2605.20520&json=true","fetch_graph":"https://pith.science/api/pith-number/QIAYVDZQMYBWLTOL4RQNEUEWRW/graph.json","fetch_events":"https://pith.science/api/pith-number/QIAYVDZQMYBWLTOL4RQNEUEWRW/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/QIAYVDZQMYBWLTOL4RQNEUEWRW/action/timestamp_anchor","attest_storage":"https://pith.science/pith/QIAYVDZQMYBWLTOL4RQNEUEWRW/action/storage_attestation","attest_author":"https://pith.science/pith/QIAYVDZQMYBWLTOL4RQNEUEWRW/action/author_attestation","sign_citation":"https://pith.science/pith/QIAYVDZQMYBWLTOL4RQNEUEWRW/action/citation_signature","submit_replication":"https://pith.science/pith/QIAYVDZQMYBWLTOL4RQNEUEWRW/action/replication_record"}},"created_at":"2026-05-21T01:04:40.814332+00:00","updated_at":"2026-05-21T01:04:40.814332+00:00"}