{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:VFJLSOAM3CGNICKSZE24KXGM3D","short_pith_number":"pith:VFJLSOAM","canonical_record":{"source":{"id":"2407.11691","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2024-07-16T13:06:15Z","cross_cats_sorted":[],"title_canon_sha256":"3c3612b87ba498f53538166b0efa42399b4910ef683e8ae122d8724d4fd00ef5","abstract_canon_sha256":"d853d3f0fce3a32c55a3d1be00a278b9466d97b7839128c7d59dfd1198be2ded"},"schema_version":"1.0"},"canonical_sha256":"a952b9380cd88cd40952c935c55cccd8d767bf1b950cd984570d4461edd29314","source":{"kind":"arxiv","id":"2407.11691","version":4},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.11691","created_at":"2026-07-05T12:00:43Z"},{"alias_kind":"arxiv_version","alias_value":"2407.11691v4","created_at":"2026-07-05T12:00:43Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.11691","created_at":"2026-07-05T12:00:43Z"},{"alias_kind":"pith_short_12","alias_value":"VFJLSOAM3CGN","created_at":"2026-07-05T12:00:43Z"},{"alias_kind":"pith_short_16","alias_value":"VFJLSOAM3CGNICKS","created_at":"2026-07-05T12:00:43Z"},{"alias_kind":"pith_short_8","alias_value":"VFJLSOAM","created_at":"2026-07-05T12:00:43Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:VFJLSOAM3CGNICKSZE24KXGM3D","target":"record","payload":{"canonical_record":{"source":{"id":"2407.11691","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2024-07-16T13:06:15Z","cross_cats_sorted":[],"title_canon_sha256":"3c3612b87ba498f53538166b0efa42399b4910ef683e8ae122d8724d4fd00ef5","abstract_canon_sha256":"d853d3f0fce3a32c55a3d1be00a278b9466d97b7839128c7d59dfd1198be2ded"},"schema_version":"1.0"},"canonical_sha256":"a952b9380cd88cd40952c935c55cccd8d767bf1b950cd984570d4461edd29314","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T12:00:43.451090Z","signature_b64":"pHrZS7WqeCTRdd4eaHUjZtO5kr7TufIDGWyd3k1ZKHvBEbhW4kPJkoSuQta1nreHBn0GbsKaecUsDZut7vT8AQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a952b9380cd88cd40952c935c55cccd8d767bf1b950cd984570d4461edd29314","last_reissued_at":"2026-07-05T12:00:43.450580Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T12:00:43.450580Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2407.11691","source_version":4,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T12:00:43Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Yo+uSsErLJD6duQOcITt/uVW60zEd0yaHaGNgDtmlVlOvM0yV7GHW5yuV1DGk01u+qMz0S5vQNA8AinMo5O2Bg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-13T02:53:01.372918Z"},"content_sha256":"03e149bceb4e60d45e8cc80084ecca848c9e5bf6005d11940afa80d84ebf5c68","schema_version":"1.0","event_id":"sha256:03e149bceb4e60d45e8cc80084ecca848c9e5bf6005d11940afa80d84ebf5c68"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:VFJLSOAM3CGNICKSZE24KXGM3D","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"VLMEvalKit: An Open-Source Toolkit for Evaluating Large Multi-Modality Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CV","authors_text":"Amit Agarwal, Chaoyou Fu, Dahua Lin, Enxin Song, Hailong Sun, Haodong Duan, Jiaqi Wang, Jixuan Chen, Junbo Cui, Junming Yang, Kai Chen, Lin Chen, Mo Li, Pan Zhang, Peiheng Zhou, Shengyuan Ding, Shiyin Lu, Song Mao, Tack Hwa Wong, Tianhao Liang, Weiyun Wang, Xiangyu Zhao, Xiaoyi Dong, Xiaozhe Li, Xinyu Fang, Yifan Zhang, Yuan Liu, Yubo Ma, Yuhang Zang, Yuxuan Qiao, Zhe Chen, Zicheng Zhang","submitted_at":"2024-07-16T13:06:15Z","abstract_excerpt":"We present VLMEvalKit: an open-source toolkit for evaluating large multi-modality models based on PyTorch. The toolkit aims to provide a user-friendly and comprehensive framework for researchers and developers to evaluate existing multi-modality models and publish reproducible evaluation results. In VLMEvalKit, we implement over 200+ different large multi-modality models, including both proprietary APIs and open-source models, as well as more than 80 different multi-modal benchmarks. By implementing a single interface, new models can be easily added to the toolkit, while the toolkit automatica"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.11691","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2407.11691/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T12:00:43Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"GSFwSXRwSDEsJ+AZVL+BlYr+uvkZB3O4BuuyY1bkcYXisruL0Og349IY2rySmDjBv+cGRbX+CaKC/kLDiX8xAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-13T02:53:01.373593Z"},"content_sha256":"8176f0e0520bffe4fdc672ee2ae6c3075b936fb5b3980220ab9ac55d7776522b","schema_version":"1.0","event_id":"sha256:8176f0e0520bffe4fdc672ee2ae6c3075b936fb5b3980220ab9ac55d7776522b"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/VFJLSOAM3CGNICKSZE24KXGM3D/bundle.json","state_url":"https://pith.science/pith/VFJLSOAM3CGNICKSZE24KXGM3D/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/VFJLSOAM3CGNICKSZE24KXGM3D/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-13T02:53:01Z","links":{"resolver":"https://pith.science/pith/VFJLSOAM3CGNICKSZE24KXGM3D","bundle":"https://pith.science/pith/VFJLSOAM3CGNICKSZE24KXGM3D/bundle.json","state":"https://pith.science/pith/VFJLSOAM3CGNICKSZE24KXGM3D/state.json","well_known_bundle":"https://pith.science/.well-known/pith/VFJLSOAM3CGNICKSZE24KXGM3D/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:VFJLSOAM3CGNICKSZE24KXGM3D","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"d853d3f0fce3a32c55a3d1be00a278b9466d97b7839128c7d59dfd1198be2ded","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2024-07-16T13:06:15Z","title_canon_sha256":"3c3612b87ba498f53538166b0efa42399b4910ef683e8ae122d8724d4fd00ef5"},"schema_version":"1.0","source":{"id":"2407.11691","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.11691","created_at":"2026-07-05T12:00:43Z"},{"alias_kind":"arxiv_version","alias_value":"2407.11691v4","created_at":"2026-07-05T12:00:43Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.11691","created_at":"2026-07-05T12:00:43Z"},{"alias_kind":"pith_short_12","alias_value":"VFJLSOAM3CGN","created_at":"2026-07-05T12:00:43Z"},{"alias_kind":"pith_short_16","alias_value":"VFJLSOAM3CGNICKS","created_at":"2026-07-05T12:00:43Z"},{"alias_kind":"pith_short_8","alias_value":"VFJLSOAM","created_at":"2026-07-05T12:00:43Z"}],"graph_snapshots":[{"event_id":"sha256:8176f0e0520bffe4fdc672ee2ae6c3075b936fb5b3980220ab9ac55d7776522b","target":"graph","created_at":"2026-07-05T12:00:43Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2407.11691/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We present VLMEvalKit: an open-source toolkit for evaluating large multi-modality models based on PyTorch. The toolkit aims to provide a user-friendly and comprehensive framework for researchers and developers to evaluate existing multi-modality models and publish reproducible evaluation results. In VLMEvalKit, we implement over 200+ different large multi-modality models, including both proprietary APIs and open-source models, as well as more than 80 different multi-modal benchmarks. By implementing a single interface, new models can be easily added to the toolkit, while the toolkit automatica","authors_text":"Amit Agarwal, Chaoyou Fu, Dahua Lin, Enxin Song, Hailong Sun, Haodong Duan, Jiaqi Wang, Jixuan Chen, Junbo Cui, Junming Yang, Kai Chen, Lin Chen, Mo Li, Pan Zhang, Peiheng Zhou, Shengyuan Ding, Shiyin Lu, Song Mao, Tack Hwa Wong, Tianhao Liang, Weiyun Wang, Xiangyu Zhao, Xiaoyi Dong, Xiaozhe Li, Xinyu Fang, Yifan Zhang, Yuan Liu, Yubo Ma, Yuhang Zang, Yuxuan Qiao, Zhe Chen, Zicheng Zhang","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2024-07-16T13:06:15Z","title":"VLMEvalKit: An Open-Source Toolkit for Evaluating Large Multi-Modality Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.11691","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:03e149bceb4e60d45e8cc80084ecca848c9e5bf6005d11940afa80d84ebf5c68","target":"record","created_at":"2026-07-05T12:00:43Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"d853d3f0fce3a32c55a3d1be00a278b9466d97b7839128c7d59dfd1198be2ded","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CV","submitted_at":"2024-07-16T13:06:15Z","title_canon_sha256":"3c3612b87ba498f53538166b0efa42399b4910ef683e8ae122d8724d4fd00ef5"},"schema_version":"1.0","source":{"id":"2407.11691","kind":"arxiv","version":4}},"canonical_sha256":"a952b9380cd88cd40952c935c55cccd8d767bf1b950cd984570d4461edd29314","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a952b9380cd88cd40952c935c55cccd8d767bf1b950cd984570d4461edd29314","first_computed_at":"2026-07-05T12:00:43.450580Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T12:00:43.450580Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"pHrZS7WqeCTRdd4eaHUjZtO5kr7TufIDGWyd3k1ZKHvBEbhW4kPJkoSuQta1nreHBn0GbsKaecUsDZut7vT8AQ==","signature_status":"signed_v1","signed_at":"2026-07-05T12:00:43.451090Z","signed_message":"canonical_sha256_bytes"},"source_id":"2407.11691","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:03e149bceb4e60d45e8cc80084ecca848c9e5bf6005d11940afa80d84ebf5c68","sha256:8176f0e0520bffe4fdc672ee2ae6c3075b936fb5b3980220ab9ac55d7776522b"],"state_sha256":"d3aa912b5a094c23500fc52e9495b30f42cb7d5e8767a54bf43bfe7f81750545"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"AuN6p/dqxRw577ACOfzoAA9RqgNp6P7E0qd5URkMSw3hkVFtLX8SdbIo7J7w4XpQnDwKYIOj2NVxtxb0ta4KAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-13T02:53:01.377967Z","bundle_sha256":"71a77d4d96ab12b722d6e331d589bc508ad95084afa0fe07478373a54e339651"}}