{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:P3DCINMWKLGYRXKVL5SF44DALS","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"50ed8adbff73e29c7e683eb48f7db1065b559c3f18d1d9d2d6049010511a60f5","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2024-10-14T14:42:12Z","title_canon_sha256":"914e8f290bee0273cda7b425f28a78265829a8c0ff3fa0cfa57e6223918466f5"},"schema_version":"1.0","source":{"id":"2410.10563","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.10563","created_at":"2026-07-05T11:36:23Z"},{"alias_kind":"arxiv_version","alias_value":"2410.10563v3","created_at":"2026-07-05T11:36:23Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.10563","created_at":"2026-07-05T11:36:23Z"},{"alias_kind":"pith_short_12","alias_value":"P3DCINMWKLGY","created_at":"2026-07-05T11:36:23Z"},{"alias_kind":"pith_short_16","alias_value":"P3DCINMWKLGYRXKV","created_at":"2026-07-05T11:36:23Z"},{"alias_kind":"pith_short_8","alias_value":"P3DCINMW","created_at":"2026-07-05T11:36:23Z"}],"graph_snapshots":[{"event_id":"sha256:3379f86217d9f3eb75865d4e9f682bf13dac1c629524448e65a38dfd33f5e66f","target":"graph","created_at":"2026-07-05T11:36:23Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.10563/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We present MEGA-Bench, an evaluation suite that scales multimodal evaluation to over 500 real-world tasks, to address the highly heterogeneous daily use cases of end users. Our objective is to optimize for a set of high-quality data samples that cover a highly diverse and rich set of multimodal tasks, while enabling cost-effective and accurate model evaluation. In particular, we collected 505 realistic tasks encompassing over 8,000 samples from 16 expert annotators to extensively cover the multimodal task space. Instead of unifying these problems into standard multi-choice questions (like MMMU","authors_text":"Bohan Lyu, Dongfu Jiang, Hexiang Hu, Jiacheng Chen, Kai Wang, Sherman Siu, Tianhao Liang, Wang Zhu, Wenhu Chen, Xiang Yue, Xuan He, Yuan Liu, Yuansheng Ni, Yubo Wang, Zhengqing Wang, Ziyan Jiang","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2024-10-14T14:42:12Z","title":"MEGA-Bench: Scaling Multimodal Evaluation to over 500 Real-World Tasks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.10563","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:6949f53c5f8196b6df3beda88594141d0f5352290db95341b9d41d002ed39f77","target":"record","created_at":"2026-07-05T11:36:23Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"50ed8adbff73e29c7e683eb48f7db1065b559c3f18d1d9d2d6049010511a60f5","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2024-10-14T14:42:12Z","title_canon_sha256":"914e8f290bee0273cda7b425f28a78265829a8c0ff3fa0cfa57e6223918466f5"},"schema_version":"1.0","source":{"id":"2410.10563","kind":"arxiv","version":3}},"canonical_sha256":"7ec624359652cd88dd555f645e70605c86662321cf18685c1c75d4662e729558","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7ec624359652cd88dd555f645e70605c86662321cf18685c1c75d4662e729558","first_computed_at":"2026-07-05T11:36:23.000394Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:36:23.000394Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"3FYXhb3XovT5twyMoBnM/IW7f9TsVCMqgBQbR3fbTK+3IuDIl5/i+8GnQdT+q9YMM8R81pQjgOIuiXlcQJ5SDw==","signature_status":"signed_v1","signed_at":"2026-07-05T11:36:23.000887Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.10563","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:6949f53c5f8196b6df3beda88594141d0f5352290db95341b9d41d002ed39f77","sha256:3379f86217d9f3eb75865d4e9f682bf13dac1c629524448e65a38dfd33f5e66f"],"state_sha256":"9a5630647fa55b9a0ab5272b2f79ec02daa5947d8613f21bdb7bdc7964dbd63f"}