feat: notebook to convert values to rates

notebooks/Play.ipynb → notebooks/Preprocessing-Play.ipynb

@@ -34,7 +34,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -47,7 +47,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -61,7 +61,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -260,7 +260,7 @@
        "Australia                         107       3         21"
       ]
      },
-     "execution_count": 23,
+     "execution_count": 5,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -279,7 +279,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -288,9 +288,16 @@
     "pop_df = pd.read_csv(zf.open(\"API_SP.POP.TOTL_DS2_en_csv_v2_821007.csv\"), skiprows=4)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "There is 2018 pop data for all countries/regions except Eritrea"
+   ]
+  },
   {
    "cell_type": "code",
-   "execution_count": 53,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
@@ -407,13 +414,12 @@
        "[2 rows x 65 columns]"
       ]
      },
-     "execution_count": 53,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "# There is 2018 pop data for all countries/regions except Eritrea\n",
     "pop_df[pd.isna(pop_df['2018'])]"
    ]
   },
@@ -421,12 +427,12 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Fix the country/region names that differ"
+    "Fix the country/region names that differ between the World Bank population data and the JHU CSSE data."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 111,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -445,7 +451,7 @@
     "     'St. Martin (French part)': 'Saint Martin',\n",
     "     'Brunei Darussalam': 'Brunei'\n",
     "}\n",
-    "current_pop_ser = pop_df[['Country Name', '2018']].copy().replace(region_wb_jhu_map).set_index('Country Name')\n",
+    "current_pop_ser = pop_df[['Country Name', '2018']].copy().replace(region_wb_jhu_map).set_index('Country Name')['2018']\n",
     "data_pop_ser = current_pop_ser[current_pop_ser.index.isin(current_totals_df.index)]"
    ]
   },
@@ -458,7 +464,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 112,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [
     {
@@ -545,7 +551,7 @@
        "Saint Barthelemy                        1       0          0"
       ]
      },
-     "execution_count": 112,
+     "execution_count": 17,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -554,6 +560,218 @@
     "current_totals_df[current_totals_df.index.isin(data_pop_ser.index) == False]"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Compute rates per 100,000 for regions with more than 100 cases"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>confirmed</th>\n",
+       "      <th>deaths</th>\n",
+       "      <th>recovered</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>Italy</th>\n",
+       "      <td>16.794282</td>\n",
+       "      <td>1.044161</td>\n",
+       "      <td>1.198055</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Republic of Korea</th>\n",
+       "      <td>14.550136</td>\n",
+       "      <td>0.104580</td>\n",
+       "      <td>0.478355</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Iran (Islamic Republic of)</th>\n",
+       "      <td>9.831264</td>\n",
+       "      <td>0.355745</td>\n",
+       "      <td>3.338620</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Norway</th>\n",
+       "      <td>7.526810</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.018817</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Bahrain</th>\n",
+       "      <td>7.008874</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>1.401775</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Mainland China</th>\n",
+       "      <td>5.798468</td>\n",
+       "      <td>0.225169</td>\n",
+       "      <td>4.315697</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Switzerland</th>\n",
+       "      <td>5.765250</td>\n",
+       "      <td>0.035226</td>\n",
+       "      <td>0.035226</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Denmark</th>\n",
+       "      <td>4.519231</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.017249</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Spain</th>\n",
+       "      <td>3.627705</td>\n",
+       "      <td>0.074908</td>\n",
+       "      <td>0.068488</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Sweden</th>\n",
+       "      <td>3.486143</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.009820</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Singapore</th>\n",
+       "      <td>2.837546</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>1.383303</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>France</th>\n",
+       "      <td>2.663194</td>\n",
+       "      <td>0.049263</td>\n",
+       "      <td>0.017914</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Belgium</th>\n",
+       "      <td>2.337580</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.008755</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Netherlands</th>\n",
+       "      <td>2.216932</td>\n",
+       "      <td>0.023214</td>\n",
+       "      <td>0.000000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Austria</th>\n",
+       "      <td>2.057186</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.045213</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Germany</th>\n",
+       "      <td>1.756947</td>\n",
+       "      <td>0.002412</td>\n",
+       "      <td>0.021706</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Hong Kong SAR</th>\n",
+       "      <td>1.610522</td>\n",
+       "      <td>0.040263</td>\n",
+       "      <td>0.872366</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>UK</th>\n",
+       "      <td>0.574531</td>\n",
+       "      <td>0.009024</td>\n",
+       "      <td>0.027072</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>US</th>\n",
+       "      <td>0.510442</td>\n",
+       "      <td>0.017117</td>\n",
+       "      <td>0.004585</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Japan</th>\n",
+       "      <td>0.459183</td>\n",
+       "      <td>0.007903</td>\n",
+       "      <td>0.079824</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Australia</th>\n",
+       "      <td>0.428131</td>\n",
+       "      <td>0.012004</td>\n",
+       "      <td>0.084026</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Malaysia</th>\n",
+       "      <td>0.409153</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.076121</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                            confirmed    deaths  recovered\n",
+       "Italy                       16.794282  1.044161   1.198055\n",
+       "Republic of Korea           14.550136  0.104580   0.478355\n",
+       "Iran (Islamic Republic of)   9.831264  0.355745   3.338620\n",
+       "Norway                       7.526810  0.000000   0.018817\n",
+       "Bahrain                      7.008874  0.000000   1.401775\n",
+       "Mainland China               5.798468  0.225169   4.315697\n",
+       "Switzerland                  5.765250  0.035226   0.035226\n",
+       "Denmark                      4.519231  0.000000   0.017249\n",
+       "Spain                        3.627705  0.074908   0.068488\n",
+       "Sweden                       3.486143  0.000000   0.009820\n",
+       "Singapore                    2.837546  0.000000   1.383303\n",
+       "France                       2.663194  0.049263   0.017914\n",
+       "Belgium                      2.337580  0.000000   0.008755\n",
+       "Netherlands                  2.216932  0.023214   0.000000\n",
+       "Austria                      2.057186  0.000000   0.045213\n",
+       "Germany                      1.756947  0.002412   0.021706\n",
+       "Hong Kong SAR                1.610522  0.040263   0.872366\n",
+       "UK                           0.574531  0.009024   0.027072\n",
+       "US                           0.510442  0.017117   0.004585\n",
+       "Japan                        0.459183  0.007903   0.079824\n",
+       "Australia                    0.428131  0.012004   0.084026\n",
+       "Malaysia                     0.409153  0.000000   0.076121"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "current_per_100000_df = current_totals_df[current_totals_df['confirmed'] > 100]\n",
+    "current_per_100000_df = current_per_100000_df.div(data_pop_ser, 'index').mul(100000).dropna()\n",
+    "current_per_100000_df.sort_values('confirmed', ascending=False)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

notebooks/ToRates.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Convert Series to Rates per 100,000"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "parameters"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "ts_folder = \"../data/covid-19_jhu-csse/\"\n",
+    "wb_path = \"../data/worldbank/SP.POP.TOTL.zip\"\n",
+    "out_folder = None\n",
+    "PAPERMILL_OUTPUT_PATH = None"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "tags": [
+     "parameters"
+    ]
+   },
+   "source": [
+    "## Read in JHU CSSE data\n",
+    "\n",
+    "I will switch to [xarray](http://xarray.pydata.org/en/stable/), but ATM, it's easier like this..."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def read_jhu_covid_region_df(name):\n",
+    "    filename = os.path.join(ts_folder, f\"time_series_19-covid-{name}.csv\")\n",
+    "    df = pd.read_csv(filename)\n",
+    "    df = df.set_index(['Country/Region', 'Province/State', 'Lat', 'Long'])\n",
+    "    df.columns = pd.to_datetime(df.columns)\n",
+    "    region_df = df.groupby(level='Country/Region').sum()\n",
+    "    loc_df = df.reset_index([2,3]).groupby(level='Country/Region').mean()[['Long', 'Lat']]\n",
+    "    return region_df.join(loc_df).set_index(['Long', 'Lat'], append=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "frames_map = {\n",
+    "    \"confirmed\": read_jhu_covid_region_df(\"Confirmed\"),\n",
+    "    \"deaths\": read_jhu_covid_region_df(\"Deaths\"),\n",
+    "    \"recovered\": read_jhu_covid_region_df(\"Recovered\")\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "frames_map['confirmed'].sort_values(frames_map['confirmed'].columns[-1], ascending=False).head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Read in World Bank data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import zipfile\n",
+    "zf = zipfile.ZipFile(wb_path)\n",
+    "pop_df = pd.read_csv(zf.open(\"API_SP.POP.TOTL_DS2_en_csv_v2_821007.csv\"), skiprows=4)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "There is 2018 pop data for all countries/regions except Eritrea"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pop_df[pd.isna(pop_df['2018'])]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Fix the country/region names that differ between the World Bank population data and the JHU CSSE data."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "region_wb_jhu_map = {\n",
+    "    'China': 'Mainland China',\n",
+    "     'Iran, Islamic Rep.': 'Iran (Islamic Republic of)',\n",
+    "     'Korea, Rep.': 'Republic of Korea',\n",
+    "     'United States': 'US',\n",
+    "     'United Kingdom': 'UK',\n",
+    "     'Hong Kong SAR, China': 'Hong Kong SAR',\n",
+    "     'Egypt, Arab Rep.': 'Egypt',\n",
+    "     'Vietnam': 'Viet Nam',\n",
+    "     'Macao SAR, China': 'Macao SAR',\n",
+    "     'Slovak Republic': 'Slovakia',\n",
+    "     'Moldova': 'Republic of Moldova',\n",
+    "     'St. Martin (French part)': 'Saint Martin',\n",
+    "     'Brunei Darussalam': 'Brunei'\n",
+    "}\n",
+    "current_pop_ser = pop_df[['Country Name', '2018']].copy().replace(region_wb_jhu_map).set_index('Country Name')['2018']\n",
+    "data_pop_ser = current_pop_ser[current_pop_ser.index.isin(frames_map['confirmed'].index.levels[0])]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "There are some regions that we cannot resolve, but we will just ignore these."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Compute rates per 100,000 for regions"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def cases_to_rates_df(df):\n",
+    "    per_100000_df = df.reset_index([1, 2], drop=True)\n",
+    "    per_100000_df = per_100000_df.div(data_pop_ser, 'index').mul(100000).dropna()\n",
+    "    per_100000_df.index.name = 'Country/Region'\n",
+    "    return per_100000_df\n",
+    "    \n",
+    "def frames_to_rates(frames_map):\n",
+    "    return {k: cases_to_rates_df(v) for k,v in frames_map.items()}\n",
+    "\n",
+    "\n",
+    "rates_map = frames_to_rates(frames_map)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if PAPERMILL_OUTPUT_PATH:\n",
+    "    for k, v in rates_map.items():\n",
+    "        out_path = os.path.join(out_folder, f\"ts_rates_19-covid-{k}.csv\")\n",
+    "        v.reset_index().to_csv(out_path)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}