{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:X5AAW5UGLZ4SNADHDMZ3FJ2GUG","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"59fdd8afdc4d86e8cc10d2b0eec556b5ad3c6d84f4a9fa5829aa2836d89f59d9","cross_cats_sorted":["cs.CY"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2023-10-25T05:38:38Z","title_canon_sha256":"8f60bbd866564b3a98e688f58ddabc9c854fcb860717ce6ac35ebfbac1bfaa29"},"schema_version":"1.0","source":{"id":"2310.16379","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2310.16379","created_at":"2026-07-05T07:28:46Z"},{"alias_kind":"arxiv_version","alias_value":"2310.16379v2","created_at":"2026-07-05T07:28:46Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.16379","created_at":"2026-07-05T07:28:46Z"},{"alias_kind":"pith_short_12","alias_value":"X5AAW5UGLZ4S","created_at":"2026-07-05T07:28:46Z"},{"alias_kind":"pith_short_16","alias_value":"X5AAW5UGLZ4SNADH","created_at":"2026-07-05T07:28:46Z"},{"alias_kind":"pith_short_8","alias_value":"X5AAW5UG","created_at":"2026-07-05T07:28:46Z"}],"graph_snapshots":[{"event_id":"sha256:27738bad46a6544411732ec9d9789b121841514b5f8b3f7fcee22d7d67240a8c","target":"graph","created_at":"2026-07-05T07:28:46Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2310.16379/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Comprehensive and accurate evaluation of general-purpose AI systems such as large language models allows for effective mitigation of their risks and deepened understanding of their capabilities. Current evaluation methodology, mostly based on benchmarks of specific tasks, falls short of adequately assessing these versatile AI systems, as present techniques lack a scientific foundation for predicting their performance on unforeseen tasks and explaining their varying performance on specific task items or user inputs. Moreover, existing benchmarks of specific tasks raise growing concerns about th","authors_text":"David Stillwell, Fang Luo, Jose Hernandez-Orallo, Liming Jiang, Luning Sun, Xing Xie, Xiting Wang","cross_cats":["cs.CY"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2023-10-25T05:38:38Z","title":"Evaluating General-Purpose AI with Psychometrics"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.16379","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:f2614057214e62ac84334b66520d72be789ddbd313d47ead28f7951eaabbc956","target":"record","created_at":"2026-07-05T07:28:46Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"59fdd8afdc4d86e8cc10d2b0eec556b5ad3c6d84f4a9fa5829aa2836d89f59d9","cross_cats_sorted":["cs.CY"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2023-10-25T05:38:38Z","title_canon_sha256":"8f60bbd866564b3a98e688f58ddabc9c854fcb860717ce6ac35ebfbac1bfaa29"},"schema_version":"1.0","source":{"id":"2310.16379","kind":"arxiv","version":2}},"canonical_sha256":"bf400b76865e792680671b33b2a746a19494f3310d921b03d539d4595477250f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"bf400b76865e792680671b33b2a746a19494f3310d921b03d539d4595477250f","first_computed_at":"2026-07-05T07:28:46.359171Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:28:46.359171Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"nyA7MatvBBDFosMR4oDIICpU2xEAeU+2Xkgaj5pw60KCPsHxxN8k2Q/u5qpZqQ7GuOZO3ekMdhN4XoQ9o/qhCg==","signature_status":"signed_v1","signed_at":"2026-07-05T07:28:46.359740Z","signed_message":"canonical_sha256_bytes"},"source_id":"2310.16379","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:f2614057214e62ac84334b66520d72be789ddbd313d47ead28f7951eaabbc956","sha256:27738bad46a6544411732ec9d9789b121841514b5f8b3f7fcee22d7d67240a8c"],"state_sha256":"d3cff1e649036c89e5e9c09ac1ed549803fe9852d5b2ffdbbeda1af4681862b5"}