runs/download-covidtracking-data.runs.ipynb

 %% Cell type:code id: tags:
 
 ``` python
 import requests
 import os
 import pandas as pd
 ```
 
 %% Cell type:code id: tags:parameters
 
 ``` python
 out_folder = "../data/covidtracking/"
 PAPERMILL_OUTPUT_PATH = None
 ```
 
 %% Cell type:code id: tags:injected-parameters
 
 ``` python
 # Parameters
-PAPERMILL_INPUT_PATH = "/tmp/vwmf0jau/notebooks/process/download-covidtracking-data.ipynb"
+PAPERMILL_INPUT_PATH = "/tmp/g0ocf832/notebooks/process/download-covidtracking-data.ipynb"
 PAPERMILL_OUTPUT_PATH = "runs/download-covidtracking-data.runs.ipynb"
 out_folder = "data/covidtracking"
 ```
 
 %% Cell type:markdown id: tags:
 
 # Download state metadata
 
 Download a dataset of URLs for data for each US state and several territories. See [Google Doc](https://docs.google.com/spreadsheets/d/18oVRrHj3c183mHmq3m89_163yuYltLNlOmPerQ18E8w/htmlview?sle=true).
 
 %% Cell type:code id: tags:
 
 ``` python
 url = 'http://covidtracking.com/api/states/info'
 r = requests.get(url, allow_redirects=True)
 states_metadata_json = r.content
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 # save the result
 if PAPERMILL_OUTPUT_PATH:
     out_path = os.path.join(out_folder, 'states-metadata.json')
     with open(out_path, 'wb') as f:
         f.write(states_metadata_json)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 metadata_df = pd.read_json(states_metadata_json)
 print(len(metadata_df), "states and territories have metadata")
 metadata_df.head(2)
 ```
 
 %% Output
 
     56 states and territories have metadata
 
       state                                     covid19SiteOld  \
     0    AK  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...
     1    AL  http://www.alabamapublichealth.gov/infectiousd...
     
                                              covid19Site  \
     0  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...
     1  https://alpublichealth.maps.arcgis.com/apps/op...
     
                                     covid19SiteSecondary          twitter  \
     0  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...     @Alaska_DHSS
     1                                               None  @alpublichealth
     
             pui    pum                                              notes  fips  \
     0  All data  False  Total tests are taken from the annotations on ...     2
     1   No data  False  Negatives = (Totals - Positives) \nPositives o...     1
     
           name
     0   Alaska
     1  Alabama
 
 %% Cell type:markdown id: tags:
 
 # Download daily state data
 
 %% Cell type:code id: tags:
 
 ``` python
 url = 'https://covidtracking.com/api/states/daily'
 r = requests.get(url, allow_redirects=True)
 states_daily_json = r.content
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 # save the result
 if PAPERMILL_OUTPUT_PATH:
     out_path = os.path.join(out_folder, 'states-daily.json')
     with open(out_path, 'wb') as f:
         f.write(states_daily_json)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 data_df = pd.read_json(states_daily_json)
 print(len(data_df), "data points")
 data_df.head(2)
 ```
 
 %% Output
 
-    2620 data points
+    2769 data points
 
            date state  positive  negative  pending  hospitalizedCurrently  \
-    0  20200421    AK     329.0   10790.0      NaN                   42.0
-    1  20200421    AL    5231.0   43295.0      NaN                    NaN
+    0  20200423    AK     337.0   11824.0      NaN                   42.0
+    1  20200423    AL    5778.0   46863.0      NaN                    NaN
     
        hospitalizedCumulative  inIcuCurrently  inIcuCumulative  \
-    0                    36.0             NaN              NaN
-    1                   699.0             NaN            260.0
+    0                     NaN             NaN              NaN
+    1                   768.0             NaN            288.0
     
        onVentilatorCurrently  ...  hospitalized    total totalTestResults  \
-    0                    NaN  ...          36.0  11119.0          11119.0
-    1                    NaN  ...         699.0  48526.0          48526.0
+    0                    NaN  ...           NaN  12161.0          12161.0
+    1                    NaN  ...         768.0  52641.0          52641.0
     
         posNeg  fips  deathIncrease  hospitalizedIncrease  negativeIncrease  \
-    0  11119.0     2            0.0                   0.0             987.0
-    1  48526.0     1           10.0                  58.0            2420.0
+    0  12161.0     2            0.0                   0.0               0.0
+    1  52641.0     1            3.0                  38.0            3568.0
     
        positiveIncrease  totalTestResultsIncrease
-    0               8.0                     995.0
-    1             206.0                    2626.0
+    0               2.0                       2.0
+    1             313.0                    3881.0
     
     [2 rows x 25 columns]