@article{2e3254b06b7749148022136a3fbe6c47,
title = "Evaluation of an open forecasting challenge to assess skill of West Nile virus neuroinvasive disease prediction",
abstract = "Background: West Nile virus (WNV) is the leading cause of mosquito-borne illness in the continental USA. WNV occurrence has high spatiotemporal variation, and current approaches to targeted control of the virus are limited, making forecasting a public health priority. However, little research has been done to compare strengths and weaknesses of WNV disease forecasting approaches on the national scale. We used forecasts submitted to the 2020 WNV Forecasting Challenge, an open challenge organized by the Centers for Disease Control and Prevention, to assess the status of WNV neuroinvasive disease (WNND) prediction and identify avenues for improvement. Methods: We performed a multi-model comparative assessment of probabilistic forecasts submitted by 15 teams for annual WNND cases in US counties for 2020 and assessed forecast accuracy, calibration, and discriminatory power. In the evaluation, we included forecasts produced by comparison models of varying complexity as benchmarks of forecast performance. We also used regression analysis to identify modeling approaches and contextual factors that were associated with forecast skill. Results: Simple models based on historical WNND cases generally scored better than more complex models and combined higher discriminatory power with better calibration of uncertainty. Forecast skill improved across updated forecast submissions submitted during the 2020 season. Among models using additional data, inclusion of climate or human demographic data was associated with higher skill, while inclusion of mosquito or land use data was associated with lower skill. We also identified population size, extreme minimum winter temperature, and interannual variation in WNND cases as county-level characteristics associated with variation in forecast skill. Conclusions: Historical WNND cases were strong predictors of future cases with minimal increase in skill achieved by models that included other factors. Although opportunities might exist to specifically improve predictions for areas with large populations and low or high winter temperatures, areas with high case-count variability are intrinsically more difficult to predict. Also, the prediction of outbreaks, which are outliers relative to typical case numbers, remains difficult. Further improvements to prediction could be obtained with improved calibration of forecast uncertainty and access to real-time data streams (e.g. current weather and preliminary human cases). Graphical Abstract: [Figure not available: see fulltext.]",
keywords = "Calibration, Discriminatory power, Forecasting, Logarithmic score, Multi-model assessment, United States, West Nile neuroinvasive disease, West Nile virus",
author = "Holcomb, {Karen M.} and Sarabeth Mathis and Staples, {J. Erin} and Marc Fischer and Barker, {Christopher M.} and Beard, {Charles B.} and Nett, {Randall J.} and Keyel, {Alexander C.} and Matteo Marcantonio and Childs, {Marissa L.} and Gorris, {Morgan E.} and Ilia Rochlin and Marco Hamins-Pu{\'e}rtolas and Ray, {Evan L.} and Uelmen, {Johnny A.} and Nicholas DeFelice and Freedman, {Andrew S.} and Hollingsworth, {Brandon D.} and Praachi Das and Dave Osthus and Humphreys, {John M.} and Nicole Nova and Mordecai, {Erin A.} and Cohnstaedt, {Lee W.} and Devin Kirk and Kramer, {Laura D.} and Harris, {Mallory J.} and Kain, {Morgan P.} and Reed, {Emily M.X.} and Johansson, {Michael A.}",
note = "Funding Information: KMH was a NOAA-CDC climate and health postdoc supported by the NOAA—Climate Adaptation and Mitigation Program and administered by UCAR's Cooperative Programs for the Advancement of Earth System Science (CPAESS) under awards #NA16OAR4310253, #NA18OAR4310253B, and #NA20OAR4310253C. CMB acknowledges funding support from the Pacific Southwest Center of Excellence in Vector-Borne Diseases funded by the US Centers for Disease Control and Prevention (Cooperative Agreement 1U01CK000516). EAM, DK, MPK, and NN were supported by the National Institutes of Health (R35GM133439). EAM was supported by the National Science Foundation (DEB-2011147 with Fogarty International Center), the Stanford Woods Institute for the Environment, King Center on Global Development, and Center for Innovation in Global Health. MLC was supported by the Illich-Sadowsky Fellowship through the Stanford Interdisciplinary Graduate Fellowship. NN was supported by the Stanford Data Science Scholars Program and the Center for Computational, Evolutionary and Human Genomics Predoctoral Fellowship. MJH was supported by the Knight-Hennessey Scholars Program. ACK was supported by cooperative agreement 1U01CK000509-01, funded by the Centers for Disease Control and Prevention. MEG gratefully acknowledges support from a Los Alamos National Laboratory, Laboratory Directed Research and Development, Director{\textquoteright}s Postdoc Fellowship. None of the funding bodies had a role in the design of the study and collection, analysis, and interpretation of data and in writing the manuscript. The findings and conclusions in this report are those of the author(s) and do not necessarily represent the views of the Centers for Disease Control and Prevention or the Department of Health and Human Services. Publisher Copyright: {\textcopyright} 2023, This is a U.S. Government work and not under copyright protection in the US; foreign copyright protection may apply.",
year = "2023",
month = dec,
doi = "10.1186/s13071-022-05630-y",
language = "English",
volume = "16",
journal = "Parasites and Vectors",
issn = "1756-3305",
publisher = "BioMed Central Ltd.",
number = "1",
}