{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:ZKRPY5FW4KZRNALQSJREA64HKR","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"6d07b6028b9e688a0d3a4ebf78cc8109226ea37b209a2d7bc976ea139dc23405","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-12-16T05:15:43Z","title_canon_sha256":"269c0589ecd5fd0a8be29bc22773df2dea8b3badba58b8418d0d8c2e102e33cc"},"schema_version":"1.0","source":{"id":"2412.11453","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.11453","created_at":"2026-07-05T09:49:33Z"},{"alias_kind":"arxiv_version","alias_value":"2412.11453v1","created_at":"2026-07-05T09:49:33Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.11453","created_at":"2026-07-05T09:49:33Z"},{"alias_kind":"pith_short_12","alias_value":"ZKRPY5FW4KZR","created_at":"2026-07-05T09:49:33Z"},{"alias_kind":"pith_short_16","alias_value":"ZKRPY5FW4KZRNALQ","created_at":"2026-07-05T09:49:33Z"},{"alias_kind":"pith_short_8","alias_value":"ZKRPY5FW","created_at":"2026-07-05T09:49:33Z"}],"graph_snapshots":[{"event_id":"sha256:cad942565f07c8dcd4d19cc9af100e876e1dcb8014c544404d173187a1696d9e","target":"graph","created_at":"2026-07-05T09:49:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2412.11453/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"As multimodal large language models (MLLMs) gain prominence in the medical field, the need for precise evaluation methods to assess their effectiveness has become critical. While benchmarks provide a reliable means to evaluate the capabilities of MLLMs, traditional metrics like ROUGE and BLEU employed for open domain evaluation only focus on token overlap and may not align with human judgment. Although human evaluation is more reliable, it is labor-intensive, costly, and not scalable. LLM-based evaluation methods have proven promising, but to date, there is still an urgent need for open-source","authors_text":"Gerard de Melo, Liang He, Linlin Wang, Shunfan Zheng, Xiaoling Wang, Xiechi Zhang, Zhu Cao","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-12-16T05:15:43Z","title":"ACE-$M^3$: Automatic Capability Evaluator for Multimodal Medical Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.11453","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:e1c45a88f1c26912e058b94bf585c376a3fcb17a5d202279ba7da6a4cafeb283","target":"record","created_at":"2026-07-05T09:49:33Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"6d07b6028b9e688a0d3a4ebf78cc8109226ea37b209a2d7bc976ea139dc23405","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-12-16T05:15:43Z","title_canon_sha256":"269c0589ecd5fd0a8be29bc22773df2dea8b3badba58b8418d0d8c2e102e33cc"},"schema_version":"1.0","source":{"id":"2412.11453","kind":"arxiv","version":1}},"canonical_sha256":"caa2fc74b6e2b31681709262407b8754411af9bced1f3920cd90e3e4ea3b3144","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"caa2fc74b6e2b31681709262407b8754411af9bced1f3920cd90e3e4ea3b3144","first_computed_at":"2026-07-05T09:49:33.315519Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:49:33.315519Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"uKkJECqgwabZo+i+h6idB22yryLpqlDU0fLWHdV/3+80ZbjY9gbxlrn+CKQxoFoqWoj/uJW96vVnimUjyoopAg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:49:33.315936Z","signed_message":"canonical_sha256_bytes"},"source_id":"2412.11453","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:e1c45a88f1c26912e058b94bf585c376a3fcb17a5d202279ba7da6a4cafeb283","sha256:cad942565f07c8dcd4d19cc9af100e876e1dcb8014c544404d173187a1696d9e"],"state_sha256":"172de14dadf4bd223e500242f1c5055435621fbb0dbf9a73c633e0c698da8655"}