{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:6Y6XGD4XNH3IUPLIIMQU7BASZ2","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"590d4de8f8fdf9a239822b9ad966dac821a803dea59a27699d9331f7191cdde8","cross_cats_sorted":["cs.CL","cs.CV"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-12-09T18:37:14Z","title_canon_sha256":"c6f0f362f78edfeb3188f8ea0c6f16c209ea2c8d06ec94b0233ad915e25657f7"},"schema_version":"1.0","source":{"id":"2412.06745","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.06745","created_at":"2026-07-05T11:22:34Z"},{"alias_kind":"arxiv_version","alias_value":"2412.06745v2","created_at":"2026-07-05T11:22:34Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.06745","created_at":"2026-07-05T11:22:34Z"},{"alias_kind":"pith_short_12","alias_value":"6Y6XGD4XNH3I","created_at":"2026-07-05T11:22:34Z"},{"alias_kind":"pith_short_16","alias_value":"6Y6XGD4XNH3IUPLI","created_at":"2026-07-05T11:22:34Z"},{"alias_kind":"pith_short_8","alias_value":"6Y6XGD4X","created_at":"2026-07-05T11:22:34Z"}],"graph_snapshots":[{"event_id":"sha256:552bd81342306fe9140e1ce360ec88272a25f4f5d94fd30b9d7eeab0092b2e49","target":"graph","created_at":"2026-07-05T11:22:34Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2412.06745/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Traditional fixed test sets fall short in evaluating open-ended capabilities of foundation models. To address this, we propose ONEBench(OpeN-Ended Benchmarking), a new testing paradigm that consolidates individual evaluation datasets into a unified, ever-expanding sample pool. ONEBench allows users to generate custom, open-ended evaluation benchmarks from this pool, corresponding to specific capabilities of interest. By aggregating samples across test sets, ONEBench enables the assessment of diverse capabilities beyond those covered by the original test sets, while mitigating overfitting and d","authors_text":"Adhiraj Ghosh, Ameya Prabhu, Matthias Bethge, Samuel Albanie, Sebastian Dziadzio, Vishaal Udandarao","cross_cats":["cs.CL","cs.CV"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-12-09T18:37:14Z","title":"ONEBench to Test Them All: Sample-Level Benchmarking Over Open-Ended Capabilities"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.06745","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:9cb6b4f48680969ca2b1adcfabc56b39757d059b1faf7c8ffb532f89495d67f2","target":"record","created_at":"2026-07-05T11:22:34Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"590d4de8f8fdf9a239822b9ad966dac821a803dea59a27699d9331f7191cdde8","cross_cats_sorted":["cs.CL","cs.CV"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-12-09T18:37:14Z","title_canon_sha256":"c6f0f362f78edfeb3188f8ea0c6f16c209ea2c8d06ec94b0233ad915e25657f7"},"schema_version":"1.0","source":{"id":"2412.06745","kind":"arxiv","version":2}},"canonical_sha256":"f63d730f9769f68a3d6843214f8412ceb5e319e85b029b15fe6eb48818ec33fe","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f63d730f9769f68a3d6843214f8412ceb5e319e85b029b15fe6eb48818ec33fe","first_computed_at":"2026-07-05T11:22:34.637594Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:22:34.637594Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"G5RLk2GHwIXMVqUgx1mWTwQDAPgw0iFx2t/vWp1iUlGXd8ma7mOntQ2tP4K7fnw7Ovv7vGqQ1NPqq8Ed73eqCA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:22:34.638144Z","signed_message":"canonical_sha256_bytes"},"source_id":"2412.06745","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:9cb6b4f48680969ca2b1adcfabc56b39757d059b1faf7c8ffb532f89495d67f2","sha256:552bd81342306fe9140e1ce360ec88272a25f4f5d94fd30b9d7eeab0092b2e49"],"state_sha256":"05db1678780025b5166b873075a1a430249e6532ae9ba85212c3204cf43cb27f"}