{"as_of":"2026-08-11T07:27:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:e903d0598e6003e452c98164cb0b5e34088bc9820c8c1a000e8c544136e209ed","coverage":[{"denominator":24,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":24,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-10T00:16:37.811235Z","state":"measured"},{"denominator":26,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":26,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-11T06:34:44.6726+00:00","state":"measured"},{"denominator":2,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":2,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-02T09:45:26.654826Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-07-02T23:47:27.686727Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"cited_work":{"arxiv_id":"2501.18243","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2501.18243","snapshot_observed_at":"2026-07-02T23:47:27.686727Z","title":"Statistical multi-metric evaluation and visualization of llm system predictive performance.arXiv preprint arXiv:2501.18243, 2025","venue":null,"work_id":"71b84a13-3da8-4faa-9d5e-b59fe90f58be","year":2025},"citing_paper":{"arxiv_id":"2606.08679","last_updated":"2026-06-07T15:31:29Z","snapshot_observed_at":"2026-08-04T23:05:45.103241Z","submitted_at":"2026-06-07T15:31:29Z","title":"Rank Intervals for Leaderboards: A Hierarchical Framework for Model Evaluation","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-06-27T17:54:22.974336Z"},"links":{"cited_paper":"/paper/2501.18243","citing_paper":"/paper/2606.08679"},"observation_digest":"sha256:5576f7941a0d556ca2ca29c4d5d700b5611df6cfafb16d61f5024e79787b37dc","observation_id":"5e68a197-fb9c-432f-9662-1cf1e06d0e88","resolution":{"observed_at":"2026-07-02T23:47:27.688270Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.18243","snapshot_observed_at":"2026-08-02T09:45:26.654826Z","title":"Sta- tistical multi-metric evaluation and visualization of llm system predictive performance.arXiv preprint arXiv:2501.18243,","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.16259","last_updated":"2026-06-28T11:19:02Z","snapshot_observed_at":"2026-08-10T21:46:57.012531Z","submitted_at":"2026-06-28T11:19:02Z","title":"Quantifying Ranking Uncertainty in LLM Benchmarks","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-02T09:45:26.654826Z"},"links":{"cited_paper":"/paper/2501.18243","citing_paper":"/paper/2607.16259"},"observation_digest":"sha256:e63a2af5ebc4eb7e7f8a8899d3e4f70cda04215c8a773dd51c37692435680174","observation_id":"0ce2d397-8c5e-43ab-b500-c7fbe4be67dd","resolution":{"observed_at":"2026-08-02T09:45:26.654826Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2501.18243/citation-record","integrity":"/paper/2501.18243/integrity","json":"/paper/2501.18243/citation-record.json","paper":"/paper/2501.18243"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2405.13020","last_updated":"2024-05-15T11:13:39Z","snapshot_observed_at":"2026-08-10T19:48:00.306859Z","submitted_at":"2024-05-15T11:13:39Z","title":"Using Combinatorial Optimization to Design a High quality LLM Solution","version":1},"cited_work":{"arxiv_id":"2405.13020","doi":null,"metadata_source":"pith","pith_arxiv_id":"2405.13020","snapshot_observed_at":"2026-08-10T00:16:37.849670Z","title":"Using Combinatorial Optimization to Design a High quality LLM Solution","venue":"cs.CL","work_id":"7e251800-fc6f-40f4-bcef-56246996ce16","year":2024},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.732561Z"},"links":{"cited_paper":"/paper/2405.13020","citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:2eebaa714097ac618689ec9a65f2be438e8161e0c5eac87ae4f783a57c5a1b8c","observation_id":"566efa22-b0e5-4cca-b077-c4d02f84bf7f","resolution":{"observed_at":"2026-08-10T00:16:37.855138Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:38.064848Z","title":"Simultaneous confidence intervals for ranks with application to ranking institutions","venue":null,"work_id":"cec4de95-568c-4433-a5ad-c9cbef1de993","year":2022},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.736883Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:794cc6c0e09c191f5092cbbb0ef0699826b2dda326231d55759dd6c63504de9e","observation_id":"04234310-eb59-4515-bb0d-50ff4bb4e5e1","resolution":{"observed_at":"2026-08-10T00:16:38.068354Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:38.054191Z","title":"Statistical Power Analysis for the Behavioral Sciences","venue":null,"work_id":"1abbaa68-24b6-4f72-83f1-1ae3c64bd2ec","year":1988},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.740522Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:13783c684da64486067ccf3ca242ceb25e04a9d0237e597764b67a1fac25fcca","observation_id":"dbba6ae6-80c4-4a9a-aefb-711f4223e140","resolution":{"observed_at":"2026-08-10T00:16:38.057584Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:38.044098Z","title":"The spotis rank reversal free method for multi-criteria decision-making support","venue":null,"work_id":"dcd2b767-119b-45f3-bb4e-c2e0a5ce3d71","year":2020},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.743949Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:2445050647f743842cf33f5235d4611609d5ba4be1d98a276ae3d924b339fa57","observation_id":"65ecd948-64de-421c-b42e-bc66b4a359c7","resolution":{"observed_at":"2026-08-10T00:16:38.047600Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:38.033687Z","title":"CrossCodeEval : A diverse and multilingual benchmark for cross-file code completion","venue":null,"work_id":"27171c28-0614-48e7-bd07-d765e7cc2e7a","year":2024},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.747876Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:fe3962df687ba09666479f10fea68be593638aaf52200eda61866c599ca66282","observation_id":"146b089a-d610-4ffb-9ddd-3d33d9cc0478","resolution":{"observed_at":"2026-08-10T00:16:38.037414Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:38.023542Z","title":"Multiobjective optimization in river basin development","venue":null,"work_id":"17c731fb-f05f-4223-bf80-67517aec4adc","year":1980},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.751357Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:0876a29b34af89226cd2f2cbe187aaf69a7509fa1b110f76dadbe99a84164453","observation_id":"10d7a8cd-a604-4d69-8e87-deff4db695cc","resolution":{"observed_at":"2026-08-10T00:16:38.027060Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:38.013475Z","title":"Sensitivity of decisions to probability estimation errors: A reexamination","venue":null,"work_id":"d8f26478-b5bd-4422-a08f-f79dc319ecf5","year":1968},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.754939Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:7e534449c59bb134e2b5081a103a4295627c26b6ecdc2444cc396c762d449f83","observation_id":"18e66ac1-1997-42f8-af99-510f415f4646","resolution":{"observed_at":"2026-08-10T00:16:38.016971Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:38.003412Z","title":"Performances are plateauing, let's make the leaderboard steep again, 2024","venue":null,"work_id":"983c98e6-b622-47f0-8c8e-51218d4b6824","year":2024},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.758183Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:6910e23076772c4a8160603d207a637fbfae2125bcb84f235b48ea240246cc82","observation_id":"a3cb713e-bac8-4366-ba79-3b15c89ed2ac","resolution":{"observed_at":"2026-08-10T00:16:38.006862Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:37.993156Z","title":"Choosing between methods of combining-values","venue":null,"work_id":"2884c844-5f84-4cd7-83c3-ffcb04f69a70","year":2018},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.761668Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:ae694522edb50d61e72722d4e1653231f6f962b844ec4c82a497bae478db5f6e","observation_id":"3924f555-4d03-4f13-a201-3e123cb7d0b7","resolution":{"observed_at":"2026-08-10T00:16:37.996655Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:37.765034Z","title":"Methods for Multiple Attribute Decision Making, pages 58--191","venue":null,"work_id":null,"year":1981},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.765034Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:f14f5a061a02a92a8c14cc4a1431940776b7a9f03fa9aa85c80982f714958be7","observation_id":"6ead93d0-6238-4d15-90c9-47eb85b512fb","resolution":{"observed_at":"2026-08-10T00:16:37.765034Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:37.982906Z","title":"pymcdm—the universal library for solving multi-criteria decision-making problems","venue":null,"work_id":"bdc43790-4add-4c02-9c17-e05fb6fa215b","year":2023},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.768698Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:ba6fc71396924df2b1d08eb5e3553652c6e12f395ad54b6ca180e3cee4873e22","observation_id":"7d9e3fec-4b19-4467-902d-03cbffc60de0","resolution":{"observed_at":"2026-08-10T00:16:37.986406Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:37.972781Z","title":null,"venue":null,"work_id":"750c1a8d-fd5b-4865-abc5-ceda44a94fa4","year":2024},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.771984Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:add3bcfcff6b917178268f94e0c6ebe134a3994969b0c4e942662ef88e3c895e","observation_id":"fb46097e-e7f7-4c1c-833a-96b88c2b8ee0","resolution":{"observed_at":"2026-08-10T00:16:37.976123Z","resolver_source":"raw_fallback","status":"malformed_identifier"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:37.962193Z","title":null,"venue":null,"work_id":"69280ec4-70c0-4988-a37c-c29cfbcff981","year":2024},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.775570Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:5fcd7484ca1a8bcb90fa1cc9a672a29de478c386873798582855eac6bc20399b","observation_id":"80e6f8df-2f84-4966-902d-3c5a65cb3b2e","resolution":{"observed_at":"2026-08-10T00:16:37.965615Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:37.951656Z","title":"Mangiafico","venue":null,"work_id":"1957baa6-7475-496c-8a74-c3e48a5c025c","year":2016},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.778900Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:284ed02d33299813e895bc4a5b9bf545b115840b9102cf729ea87af6e18ee772","observation_id":"e0dde2ef-6946-42d1-a5ff-4b4f2838527c","resolution":{"observed_at":"2026-08-10T00:16:37.955295Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2405.04324","last_updated":"2024-05-07T13:50:40Z","snapshot_observed_at":"2026-08-09T20:15:03.728117Z","submitted_at":"2024-05-07T13:50:40Z","title":"Granite Code Models: A Family of Open Foundation Models for Code Intelligence","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.04324","snapshot_observed_at":"2026-08-10T00:16:37.782195Z","title":"Granite code models: A family of open foundation models for code intelligence","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.782195Z"},"links":{"cited_paper":"/paper/2405.04324","citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:41097ab88afbeef17829becc3e40f54d774b43f9feb937814c4f507b14e26635","observation_id":"fea60797-81dd-4a12-b480-25545207c333","resolution":{"observed_at":"2026-08-10T00:16:37.782195Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:37.940552Z","title":"New effect size rules of thumb","venue":null,"work_id":"c5e81511-35d1-4ce5-874a-eec6a5abf3d4","year":2009},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.785701Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:581a21cf84cf868258940e67eb4dc0f1da60cd9844f40551c40d256dbfb1d4ab","observation_id":"6cd96f46-fd17-454c-851c-7f8ef76e2f37","resolution":{"observed_at":"2026-08-10T00:16:37.944589Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:37.929494Z","title":"Statsmodels: Econometric and statistical modeling with python","venue":null,"work_id":"722857bc-ec02-4e6c-81f1-ffce9c3a4573","year":2010},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.788961Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:41359b7fa10188fff434c9e4e1f16181fcb4427d33ae026ecb345cfb00c4d372","observation_id":"38214852-9c4a-4136-8253-42363027f9b3","resolution":{"observed_at":"2026-08-10T00:16:37.933028Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:37.920033Z","title":"A multiple criteria decision making method based on relative value distances","venue":null,"work_id":"ca9dfc1f-8905-4c89-9dec-d187eac9c2be","year":2015},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.792056Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:d647b171a6d15ed6f2374427c76700595e5e91c12cad8027d7e25c7170f3934c","observation_id":"10c01e60-8dca-4ec2-80f1-0f73ea3f7893","resolution":{"observed_at":"2026-08-10T00:16:37.923503Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:37.910530Z","title":"Comparative analysis of some prominent mcdm methods: A case of ranking serbian banks","venue":null,"work_id":"5efad201-1d00-4f71-b6b5-e9c6927ea4ef","year":2013},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.795146Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:bcb9ee50bd6e2eb93e521a60a5a8828c0455bf7a95df9522ba01d8b1f4097ad7","observation_id":"e09f9612-cc96-4cb7-97eb-66ac0de49479","resolution":{"observed_at":"2026-08-10T00:16:37.914021Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:37.900985Z","title":"Using effect size—or why the p value is not enough","venue":null,"work_id":"c705623c-58f5-449d-9167-f1d9d7d8cc80","year":2012},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.798176Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:f926e558bc932fdae42294acbe6834e56b1df1a17b5d8d48fb5fbd0d7430d934","observation_id":"2befde40-c5ff-4ec2-b8a0-d120decf3582","resolution":{"observed_at":"2026-08-10T00:16:37.904452Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:37.891380Z","title":"Calculating and synthesizing effect sizes","venue":null,"work_id":"c8ec1e88-c14b-4d65-a764-135846acc4d3","year":2006},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.802219Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:63ab8a6585d2440351ee08658574246d056d3093ababee3de47da3939b5fcb23","observation_id":"cc56db27-137e-4e46-8edc-bc2589e04b89","resolution":{"observed_at":"2026-08-10T00:16:37.894808Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:37.805163Z","title":"Oliphant, Matt Haberland, Tyler Reddy, David Cournapeau, Evgeni Burovski, Pearu Peterson, Warren Weckesser, Jonathan Bright, St \\'e fan J","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.805163Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:28170bff0d2fc54903f8de58156c384b2238849f1f459fd59be0bb7e4b26a9df","observation_id":"ebff8b2c-07e0-4a4a-acb6-6d151fa38c95","resolution":{"observed_at":"2026-08-10T00:16:37.805163Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:37.873302Z","title":"The harmonic mean p-value for combining dependent tests","venue":null,"work_id":"5d7ece4a-d82b-43a8-b9aa-80ab0cda9bd4","year":2019},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.808144Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:c9f3030bd9d92df5785bb285aa85c59f0d0fb4b12fd3eb04880de9403cec8b62","observation_id":"616e18c3-5041-4049-9a04-c05c459df20e","resolution":{"observed_at":"2026-08-10T00:16:37.876952Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-10T00:16:37.862787Z","title":"harmonicmeanp tutorial, 2024","venue":null,"work_id":"ac5905f9-58bd-4dd0-a7db-57428e960377","year":2024},"citing_paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance","version":1},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-10T00:16:37.811235Z"},"links":{"citing_paper":"/paper/2501.18243"},"observation_digest":"sha256:d70df37b6f1e0b759f0428a64bf88eb3d7f1eda500c7dd162d6fc47eae4c3f88","observation_id":"651f04b1-56d5-49d7-bca7-d10048e76774","resolution":{"observed_at":"2026-08-10T00:16:37.866262Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-11T06:34:44.6726+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2501.18243","last_updated":"2025-01-30T10:21:10Z","latest_version":1,"primary_category":"stat.AP","snapshot_observed_at":"2026-08-10T00:09:56.685976Z","submitted_at":"2025-01-30T10:21:10Z","title":"Statistical multi-metric evaluation and visualization of LLM system predictive performance"},"reference_resolution":{"displayed":24,"state_counts":{"malformed_identifier":1,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":4,"verified_exact":1,"verified_fuzzy":18},"total_outbound_references":24},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-11T06:34:44.6726+00:00","source":"crossref"},{"observed_at":"2026-08-11T06:34:36.301508+00:00","source":"retraction_watch"}],"thesis":"As of 11 August 2026, this Paper Citation Record lists 24 of 24 outbound references and 2 inbound Pith citation observations for arXiv:2501.18243."}