{"as_of":"2026-08-09T05:07:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:96cab374ca877b546703452f29eb6473b0fab5b150dd99c97d9010051a42ca5e","coverage":[{"denominator":49,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":49,"source":"paper_references, paper_reference_links","source_observed_at":"2026-07-14T13:17:55.700527Z","state":"measured"},{"denominator":49,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":49,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-08T06:32:00.761636+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2607.10240/citation-record","integrity":"/paper/2607.10240/integrity","json":"/paper/2607.10240/citation-record.json","paper":"/paper/2607.10240"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:ccf18a50968b1129ae707138688720cc55ce7178d5f7187985acd8d290824263","observation_id":"26b102c3-a43b-4ec0-8008-0eedf637e57a","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"Lawrence Zitnick, and Devi Parikh","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:fe1569612b64c6105aaec9662024cea697e89bcf582e7530f6dd43afdcd3db22","observation_id":"2a441827-94cb-4111-8217-e72426d2217a","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"InProceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:23b6a32986c2c4fa869a69efceed33ac8af8b67eb9be6f78988ae4a75d9860e9","observation_id":"a8fe680c-579b-4ba4-9c6c-de4ee052b74d","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.13923","last_updated":"2025-02-19T18:00:14Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2025-02-19T18:00:14Z","title":"Qwen2.5-VL Technical Report","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.13923","snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"cited_paper":"/paper/2502.13923","citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:3a2f5e41e9169710850ac0ba283fc775323f712e0ea38e9fea5e00123591f077","observation_id":"174b0ce4-7f06-4366-a379-ea746e09bf3c","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:3f90afbd453250d15758f4f14a1926709fafec851629e293f3d66c2e1d58c1f9","observation_id":"978781ae-ec38-4684-8736-ef0e8b22037e","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:565225236c0a2f1b7619c4ff0719c5b42f294dda7d779361d5c4eb45809a2f29","observation_id":"cead67fd-ee82-42c8-b0e2-61de3fb884f2","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:f25c70a13367281a27186359d702f790a353658001774e9dd8bcbc3a099ba811","observation_id":"1158f281-8989-4d84-8274-fa8c3a7ed59a","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:bd5b794fabca754680872cbc8d11e919a7716be78f9517f49c49285fce674209","observation_id":"1dd1c501-ba03-440e-b893-ced9f776338a","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"In Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:bebd3f81ce21c226804df293897b34972f4f2104d2828f608d9e0b74c23c15bf","observation_id":"42020f16-b031-47c0-929b-3d1dd330a494","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:6f6a0a61e57c1347b58420fce67e97ede4d93bfdffa942620c58c31b25e5a301","observation_id":"af2a2ebd-ed25-44e5-9521-cf5b8abcbb19","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2503.19786","last_updated":"2025-03-25T15:52:34Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2025-03-25T15:52:34Z","title":"Gemma 3 Technical Report","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.19786","snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"cited_paper":"/paper/2503.19786","citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:31c206b337048c7321b9e2676c01411ebd280e54625184cb8f58d74963329cb0","observation_id":"5793357f-1a46-499b-baf2-d75f22d36c3f","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.07270","last_updated":"2024-05-05T20:34:28Z","snapshot_observed_at":"2026-08-07T17:45:50.717841Z","submitted_at":"2024-02-11T18:26:18Z","title":"Open-ended VQA benchmarking of Vision-Language models by exploiting Classification datasets and their semantic hierarchy","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.07270","snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"Bravo, and Thomas Brox","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"cited_paper":"/paper/2402.07270","citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:27522fbb54740d618853af6a7ea3a190f5a6f80d32c7523867cea93e8868b981","observation_id":"b0f029c4-825f-48b4-a21b-d377385af2f8","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.21783","last_updated":"2024-11-23T23:27:33Z","snapshot_observed_at":"2026-07-06T18:55:11.576666Z","submitted_at":"2024-07-31T17:54:27Z","title":"The Llama 3 Herd of Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.21783","snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"cited_paper":"/paper/2407.21783","citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:e5f069ea2da85b0521cd85a39cc8d12c83b30fdc51683857e829b2c8d6637afc","observation_id":"bf5de91a-d61c-46bf-b96d-82e8b784efdf","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:66d0c06c02d234a82c86a12a26c4d24d13f81b6f8f2adedde1432d2216071803","observation_id":"1ca64184-b7b6-487a-bd76-4307e054e4e5","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:c095bccc8ae5abc8186fb42d6307eb3f55fd82a221cd625de683f52c7f24b3c4","observation_id":"797f9128-dc06-4d91-8294-3e45b0f70872","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"InFind- ings of the Association for Computational Linguistics: EMNLP","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:9a71e5099c769138408d882b0e5b25eb1305c6cd0710873f6586dcba6bd5659c","observation_id":"8f6d4e81-69c7-4d8f-b77f-7f7249b814fc","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:9d25000a83768ac42f5805d7abd8af5ee993b882db03a39a67ef5e01816c1f77","observation_id":"09c494c6-cb8f-44ad-ad6b-52d062b925d3","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:2a785a4d8535ffd2ce01d1a384208de36ad6e312d9d8e90f0fad169be0e321d9","observation_id":"ea7664fd-f28a-41ba-afc8-d2e8353bafab","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:f523a7a7eecc6909a34fd57d001a87f5ff8bd45edd89a753ac8a57663abc11fe","observation_id":"9f08e2be-a55b-4834-b447-9c027527f1e1","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:214907e2ab7d4102c94536500c220005dc8340fda3930d2a147f390ab7d217d1","observation_id":"9819454d-ae5e-4800-b61d-6c7df4bda32f","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"Gonzalez, Hao Zhang, and Ion Stoica","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:d42c3464659125b95ecc05d741988c2266fe9ebb40d99af21afeb48d51d4c765","observation_id":"3aecee85-bf54-4213-8cf2-59105151aa42","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:18c14991aed7c3ec4715ac401d080c1617f62d69871eac270872ac57f2a3cdd8","observation_id":"c5df8434-33d6-46e0-843b-502f1b615588","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2013},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:6c24a1b9bf21053f242bafee858b63fd0ec4083ea8011ae0a6d4db512d090b2e","observation_id":"05614de5-8417-40e8-bdae-395a180d34b2","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:df76b409318b530070654cb6d7876aaf6a76e676edf69ff06e7bc247d6a72a90","observation_id":"eb0f4217-f738-4f4d-be83-00b99dc2619e","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:1065b3bca7f86ab6ef8379e19cc8d317a725b180922f6ed1f9b258d918141642","observation_id":"cd3131db-2d22-4da2-821a-034e5b8c8f5a","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:00c67e02b24ace32fdefbfb01d51e71f97a60176ca092d8832b633a00175f807","observation_id":"c62d589b-bdb4-43d0-bbee-1b4e4fc9d59a","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:da76153c1870c51cb0e74e08b665af9826a711c5e532911e57702ff4c9e44ce2","observation_id":"6086f200-4f9d-4d24-a3b4-46876332ce3e","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:c622987d85c044354f819cdf24d6b27e4beae8f8776d04c1670278023fa3a464","observation_id":"0de7136f-b488-49f6-abbb-cd4c04b32719","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:8dd7f25b0bc7f3ad630b3ffebbd38171121d8818ccceeb7d960c85bed74d7063","observation_id":"6ab9deb0-9a4b-4c98-b7cd-9c8155674bfe","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:46ba75f8bdd8b9eba3b467d76cfdc9ffc50c4c284a90cdd0cf6fd2a23fbf8458","observation_id":"6d1722f6-70d4-4508-b0fd-39718cd21532","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:2efada9055768fc94c38c50807944c3fd6ce5e768481f03959acf9068fb1c17c","observation_id":"3401c0ef-217c-40bb-a986-7f33841017bf","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:9fcf7ae655813392db1f8ebd5ac276094e6e34b683192b2a24978d397a15ece4","observation_id":"868d5bfd-0ceb-44ce-8c61-866da95c08d4","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:73a11b3746917484f40cbcede62089e2eb765888b183db77856dc4abc2deaa1c","observation_id":"d2292abf-2aca-4673-ade5-5e44490ebcb8","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.03848","last_updated":"2025-04-22T14:11:50Z","snapshot_observed_at":"2026-08-06T01:28:29.570476Z","submitted_at":"2024-02-06T09:50:08Z","title":"ANLS* -- A Universal Document Processing Metric for Generative Large Language Models","version":10},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.03848","snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"cited_paper":"/paper/2402.03848","citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:7628a04f5acd3ee141cbd3cd88d8859c6c73476c054f84a8dc1775f9066f90da","observation_id":"733539bd-7344-42e5-bdfb-3d6a08b28efc","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2016},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:26b88ff068f981cbf0ac1b78ac36f802c99505172e160eb89d7b54e906aa0ea0","observation_id":"8624e63b-b5ef-40bd-8832-895beea429ba","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:87c9d121b117862fdd6da9ec3134cd9a0bf79f0930e5faf99928a8591706690b","observation_id":"8000e368-7a7b-4285-8c2a-1dd293331a3a","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:3cf1778f722cbebc08c07d3265590a9ce2b580b785be77fc810d82226b249c66","observation_id":"2a0b000b-fd5c-40cf-82c9-914acbfca9df","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"Le, Ed H","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:49b4378f7e6d93e0e9536acc80df8632a0bdafcc8479815561713ce1f0fd3e72","observation_id":"2ec11b24-cda5-46bb-9021-32a9bbafc844","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"InProceedings of the International Conference on Learning Representations (ICLR)","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:10b879029d1f76d3e281d70909a07ece85ec6ec77507cf8717088e2353c9cd96","observation_id":"da78a49b-cfd7-4d62-b0f8-ee25765bdefb","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:9f37cbf24b5dc158527106dec0628b1d4bdd7a85f7ae8c0e493c2a5aa481c881","observation_id":"8c253bc3-e8ad-46cf-acb1-07b3173a5ed9","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2013},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:d33b29a0cae6638b1c58f4fc6eca0f8491489411a1bb0c10b076fdf2cd9f4d03","observation_id":"cb3fcd5e-5456-40ce-8d82-dd65fe8e264d","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:01bf7488cd4bc44329781f549dc995a4ecabc9abf12b674c37a85b5d9c64dab9","observation_id":"3d09161f-6065-4973-905d-f8c4b8e4e035","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"Weld, and Jonathan Berant","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:60345b19f575294e55dca51bd6ee05079c61fc107977b7fdec5e54b10b5a74ce","observation_id":"2ec93f55-ccb3-43d6-90dd-c4e8e7199fd0","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:d4ef53afe2b801f4c75872b84888e677ab75b7786b80d28cc56624122a05a440","observation_id":"190bb2dc-22b6-4241-bf9e-166d1a6817ea","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2606.06865","last_updated":"2026-06-05T03:24:38Z","snapshot_observed_at":"2026-07-06T23:46:37.903443Z","submitted_at":"2026-06-05T03:24:38Z","title":"Are Large Language Models Suitable for Graph Computation? Progress and Prospects","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2606.06865","snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"cited_paper":"/paper/2606.06865","citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:043572570d900ea22992a0f1b7e0cf785ec3fd344dff8a9385d42a1c1f54521f","observation_id":"178c6799-d934-45f7-a11f-291314743a04","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"Xing, et al","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:3ee9c61e2f048171190f9e76fa0ddcdca10989d6a25cb105444cc9c35960e754","observation_id":"ee4cec9d-a0bb-4d81-978f-b47215c88469","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:de492040485c29cdcc1a840c86df98d6a75b4200470c2a7fb104f3c854f14366","observation_id":"73a301a1-6b82-4671-9999-27aa813222c7","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":null,"venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:2b67bb1b04e78ab73c6587cdf1d62329d0973f28fc9e2934801f41b089db79f7","observation_id":"5bed46e7-fdf3-4f39-a509-fc3466cd52fc","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2504.10479","last_updated":"2025-04-19T03:47:21Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2025-04-14T17:59:25Z","title":"InternVL3: Exploring Advanced Training and Test-Time Recipes for Open-Source Multimodal Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2504.10479","snapshot_observed_at":"2026-07-14T13:17:55.700527Z","title":"official reversal","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks","version":1},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-07-14T13:17:55.700527Z"},"links":{"cited_paper":"/paper/2504.10479","citing_paper":"/paper/2607.10240"},"observation_digest":"sha256:9f51ba4949450ba56ee8b517bbb48e25ec80f261b784e5a31b2efa1a478c35e8","observation_id":"c177fff9-953b-4a7a-a737-af2d8a944b0a","resolution":{"observed_at":"2026-07-14T13:17:55.700527Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2607.10240","last_updated":"2026-07-11T10:01:29Z","latest_version":1,"primary_category":"cs.CV","snapshot_observed_at":"2026-08-06T06:21:29.752701Z","submitted_at":"2026-07-11T10:01:29Z","title":"What Does Your Short-Answer VQA Score Actually Measure? Evaluator-Dependent Instability in Multimodal Short-Answer Benchmarks"},"reference_resolution":{"displayed":49,"state_counts":{"malformed_identifier":2,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":47,"verified_exact":0,"verified_fuzzy":0},"total_outbound_references":49},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-08T06:32:00.761636+00:00","source":"crossref"},{"observed_at":"2026-08-08T06:31:55.24221+00:00","source":"retraction_watch"}],"thesis":"As of 9 August 2026, this Paper Citation Record lists 49 of 49 outbound references and 0 inbound Pith citation observations for arXiv:2607.10240."}