renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json

3e87830f · CR (covid cron) · renku 0.10.2 · 3e1f0d42 · 3e87830f · 3e87830f
Commit 3e87830f authored 4 years ago by CR (covid cron) Committed by renku 0.10.2 4 years ago
--- a/.renku/workflow/65a9c0cbf9a7441a81ca03c877e1f779.cwl
+++ b/.renku/workflow/65a9c0cbf9a7441a81ca03c877e1f779.cwl
+class: Workflow
+cwlVersion: v1.0
+hints: []
+inputs:
+  input_1:
+    default: states-metadata.json
+    streamable: false
+    type: string
+  input_2:
+    default: states-daily.json
+    streamable: false
+    type: string
+  input_3:
+    default: out_folder
+    streamable: false
+    type: string
+  input_4:
+    default: data/covidtracking
+    streamable: false
+    type: string
+  input_5:
+    default:
+      class: File
+      path: ../../notebooks/process/download-covidtracking-data.ipynb
+    streamable: false
+    type: File
+  input_6:
+    default: runs/download-covidtracking-data.runs.ipynb
+    streamable: false
+    type: string
+outputs:
+  output_1:
+    outputSource: step_3/output_0
+    streamable: false
+    type: File
+  output_2:
+    outputSource: step_3/output_1
+    streamable: false
+    type: Directory
+requirements: []
+steps:
+  step_1:
+    in:
+      filename: input_1
+      input_directory: step_3/output_1
+    out:
+    - output_file
+    run:
+      arguments: []
+      baseCommand:
+      - 'true'
+      class: CommandLineTool
+      cwlVersion: v1.0
+      hints: []
+      inputs:
+        filename:
+          default: states-metadata.json
+          streamable: false
+          type: string
+        input_directory:
+          streamable: false
+          type: Directory
+      outputs:
+        output_file:
+          outputBinding:
+            glob: $(inputs.filename)
+          streamable: false
+          type: File
+      permanentFailCodes: []
+      requirements:
+      - &id001
+        class: InlineJavascriptRequirement
+      - &id002
+        class: InitialWorkDirRequirement
+        listing: $(inputs.input_directory.listing)
+      successCodes: []
+      temporaryFailCodes: []
+  step_2:
+    in:
+      filename: input_2
+      input_directory: step_3/output_1
+    out:
+    - output_file
+    run:
+      arguments: []
+      baseCommand:
+      - 'true'
+      class: CommandLineTool
+      cwlVersion: v1.0
+      hints: []
+      inputs:
+        filename:
+          default: states-daily.json
+          streamable: false
+          type: string
+        input_directory:
+          streamable: false
+          type: Directory
+      outputs:
+        output_file:
+          outputBinding:
+            glob: $(inputs.filename)
+          streamable: false
+          type: File
+      permanentFailCodes: []
+      requirements:
+      - *id001
+      - *id002
+      successCodes: []
+      temporaryFailCodes: []
+  step_3:
+    in:
+      input_1: input_3
+      input_2: input_4
+      input_3: input_5
+      input_4: input_6
+    out:
+    - output_0
+    - output_1
+    run: a17d560c41a54f5aa307ce5f3c5effe5_papermill.cwl
--- a/data/covidtracking/states-daily.json
+++ b/data/covidtracking/states-daily.json
--- a/data/covidtracking/states-metadata.json
+++ b/data/covidtracking/states-metadata.json
--- a/runs/download-covidtracking-data.runs.ipynb
+++ b/runs/download-covidtracking-data.runs.ipynb
@@ -5,10 +5,10 @@
   "execution_count": 1,
   "metadata": {
    "papermill": {
-     "duration": 2.580306,
-     "end_time": "2020-04-17T07:25:14.828781",
+     "duration": 0.532269,
+     "end_time": "2020-04-18T18:11:05.616063",
     "exception": false,
-     "start_time": "2020-04-17T07:25:12.248475",
+     "start_time": "2020-04-18T18:11:05.083794",
     "status": "completed"
    },
    "tags": []
@@ -25,10 +25,10 @@
   "execution_count": 2,
   "metadata": {
    "papermill": {
-     "duration": 0.026104,
-     "end_time": "2020-04-17T07:25:14.871596",
+     "duration": 0.018483,
+     "end_time": "2020-04-18T18:11:05.645046",
     "exception": false,
-     "start_time": "2020-04-17T07:25:14.845492",
+     "start_time": "2020-04-18T18:11:05.626563",
     "status": "completed"
    },
    "tags": [
@@ -46,10 +46,10 @@
   "execution_count": 3,
   "metadata": {
    "papermill": {
-     "duration": 0.026826,
-     "end_time": "2020-04-17T07:25:14.911972",
+     "duration": 0.019301,
+     "end_time": "2020-04-18T18:11:05.674172",
     "exception": false,
-     "start_time": "2020-04-17T07:25:14.885146",
+     "start_time": "2020-04-18T18:11:05.654871",
     "status": "completed"
    },
    "tags": [
@@ -59,7 +59,7 @@
   "outputs": [],
   "source": [
    "# Parameters\n",
-    "PAPERMILL_INPUT_PATH = \"/tmp/pnqb5z0q/notebooks/process/download-covidtracking-data.ipynb\"\n",
+    "PAPERMILL_INPUT_PATH = \"/tmp/lvwjkfpt/notebooks/process/download-covidtracking-data.ipynb\"\n",
    "PAPERMILL_OUTPUT_PATH = \"runs/download-covidtracking-data.runs.ipynb\"\n",
    "out_folder = \"data/covidtracking\"\n"
   ]
@@ -68,10 +68,10 @@
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
-     "duration": 0.007223,
-     "end_time": "2020-04-17T07:25:14.929276",
+     "duration": 0.007258,
+     "end_time": "2020-04-18T18:11:05.690126",
     "exception": false,
-     "start_time": "2020-04-17T07:25:14.922053",
+     "start_time": "2020-04-18T18:11:05.682868",
     "status": "completed"
    },
    "tags": []
@@ -87,10 +87,10 @@
   "execution_count": 4,
   "metadata": {
    "papermill": {
-     "duration": 0.20702,
-     "end_time": "2020-04-17T07:25:15.142890",
+     "duration": 0.186003,
+     "end_time": "2020-04-18T18:11:05.883651",
     "exception": false,
-     "start_time": "2020-04-17T07:25:14.935870",
+     "start_time": "2020-04-18T18:11:05.697648",
     "status": "completed"
    },
    "tags": []
@@ -107,10 +107,10 @@
   "execution_count": 5,
   "metadata": {
    "papermill": {
-     "duration": 0.024628,
-     "end_time": "2020-04-17T07:25:15.178468",
+     "duration": 0.022463,
+     "end_time": "2020-04-18T18:11:05.916296",
     "exception": false,
-     "start_time": "2020-04-17T07:25:15.153840",
+     "start_time": "2020-04-18T18:11:05.893833",
     "status": "completed"
    },
    "tags": []
@@ -129,10 +129,10 @@
   "execution_count": 6,
   "metadata": {
    "papermill": {
-     "duration": 0.098225,
-     "end_time": "2020-04-17T07:25:15.284990",
+     "duration": 0.061515,
+     "end_time": "2020-04-18T18:11:05.986966",
     "exception": false,
-     "start_time": "2020-04-17T07:25:15.186765",
+     "start_time": "2020-04-18T18:11:05.925451",
     "status": "completed"
    },
    "tags": []
@@ -246,10 +246,10 @@
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
-     "duration": 0.008857,
-     "end_time": "2020-04-17T07:25:15.307561",
+     "duration": 0.008277,
+     "end_time": "2020-04-18T18:11:06.004318",
     "exception": false,
-     "start_time": "2020-04-17T07:25:15.298704",
+     "start_time": "2020-04-18T18:11:05.996041",
     "status": "completed"
    },
    "tags": []
@@ -263,10 +263,10 @@
   "execution_count": 7,
   "metadata": {
    "papermill": {
-     "duration": 1.815188,
-     "end_time": "2020-04-17T07:25:17.129424",
+     "duration": 0.160192,
+     "end_time": "2020-04-18T18:11:06.172926",
     "exception": false,
-     "start_time": "2020-04-17T07:25:15.314236",
+     "start_time": "2020-04-18T18:11:06.012734",
     "status": "completed"
    },
    "tags": []
@@ -283,10 +283,10 @@
   "execution_count": 8,
   "metadata": {
    "papermill": {
-     "duration": 0.030238,
-     "end_time": "2020-04-17T07:25:17.180355",
+     "duration": 0.025967,
+     "end_time": "2020-04-18T18:11:06.211115",
     "exception": false,
-     "start_time": "2020-04-17T07:25:17.150117",
+     "start_time": "2020-04-18T18:11:06.185148",
     "status": "completed"
    },
    "tags": []
@@ -305,10 +305,10 @@
   "execution_count": 9,
   "metadata": {
    "papermill": {
-     "duration": 0.1129,
-     "end_time": "2020-04-17T07:25:17.305054",
+     "duration": 0.111813,
+     "end_time": "2020-04-18T18:11:06.335104",
     "exception": false,
-     "start_time": "2020-04-17T07:25:17.192154",
+     "start_time": "2020-04-18T18:11:06.223291",
     "status": "completed"
    },
    "tags": []
@@ -318,7 +318,7 @@
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "2340 data points\n"
+      "2396 data points\n"
     ]
    },
    {
@@ -368,51 +368,51 @@
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
-       "      <td>20200416</td>\n",
+       "      <td>20200417</td>\n",
       "      <td>AK</td>\n",
-       "      <td>300.0</td>\n",
-       "      <td>8435.0</td>\n",
+       "      <td>309.0</td>\n",
+       "      <td>9141.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
-       "      <td>35.0</td>\n",
+       "      <td>36.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
-       "      <td>35.0</td>\n",
-       "      <td>8735.0</td>\n",
-       "      <td>8735.0</td>\n",
-       "      <td>8735.0</td>\n",
+       "      <td>36.0</td>\n",
+       "      <td>9450.0</td>\n",
+       "      <td>9450.0</td>\n",
+       "      <td>9450.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
-       "      <td>64.0</td>\n",
-       "      <td>7.0</td>\n",
-       "      <td>71.0</td>\n",
+       "      <td>706.0</td>\n",
+       "      <td>9.0</td>\n",
+       "      <td>715.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
-       "      <td>20200416</td>\n",
+       "      <td>20200417</td>\n",
       "      <td>AL</td>\n",
-       "      <td>4345.0</td>\n",
-       "      <td>32046.0</td>\n",
+       "      <td>4530.0</td>\n",
+       "      <td>33318.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
-       "      <td>553.0</td>\n",
+       "      <td>594.0</td>\n",
       "      <td>NaN</td>\n",
-       "      <td>227.0</td>\n",
+       "      <td>247.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
-       "      <td>553.0</td>\n",
-       "      <td>36391.0</td>\n",
-       "      <td>36391.0</td>\n",
-       "      <td>36391.0</td>\n",
+       "      <td>594.0</td>\n",
+       "      <td>37848.0</td>\n",
+       "      <td>37848.0</td>\n",
+       "      <td>37848.0</td>\n",
       "      <td>1</td>\n",
-       "      <td>12.0</td>\n",
-       "      <td>28.0</td>\n",
-       "      <td>2082.0</td>\n",
-       "      <td>232.0</td>\n",
-       "      <td>2314.0</td>\n",
+       "      <td>11.0</td>\n",
+       "      <td>41.0</td>\n",
+       "      <td>1272.0</td>\n",
+       "      <td>185.0</td>\n",
+       "      <td>1457.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
@@ -421,24 +421,24 @@
      ],
      "text/plain": [
       "       date state  positive  negative  pending  hospitalizedCurrently  \\\n",
-       "0  20200416    AK     300.0    8435.0      NaN                    NaN   \n",
-       "1  20200416    AL    4345.0   32046.0      NaN                    NaN   \n",
+       "0  20200417    AK     309.0    9141.0      NaN                    NaN   \n",
+       "1  20200417    AL    4530.0   33318.0      NaN                    NaN   \n",
       "\n",
       "   hospitalizedCumulative  inIcuCurrently  inIcuCumulative  \\\n",
-       "0                    35.0             NaN              NaN   \n",
-       "1                   553.0             NaN            227.0   \n",
+       "0                    36.0             NaN              NaN   \n",
+       "1                   594.0             NaN            247.0   \n",
       "\n",
       "   onVentilatorCurrently  ...  hospitalized    total totalTestResults  \\\n",
-       "0                    NaN  ...          35.0   8735.0           8735.0   \n",
-       "1                    NaN  ...         553.0  36391.0          36391.0   \n",
+       "0                    NaN  ...          36.0   9450.0           9450.0   \n",
+       "1                    NaN  ...         594.0  37848.0          37848.0   \n",
       "\n",
       "    posNeg  fips  deathIncrease  hospitalizedIncrease  negativeIncrease  \\\n",
-       "0   8735.0     2            0.0                   1.0              64.0   \n",
-       "1  36391.0     1           12.0                  28.0            2082.0   \n",
+       "0   9450.0     2            0.0                   1.0             706.0   \n",
+       "1  37848.0     1           11.0                  41.0            1272.0   \n",
       "\n",
       "   positiveIncrease  totalTestResultsIncrease  \n",
-       "0               7.0                      71.0  \n",
-       "1             232.0                    2314.0  \n",
+       "0               9.0                     715.0  \n",
+       "1             185.0                    1457.0  \n",
       "\n",
       "[2 rows x 25 columns]"
      ]
@@ -474,18 +474,18 @@
   "version": "3.7.3"
  },
  "papermill": {
-   "duration": 6.416916,
-   "end_time": "2020-04-17T07:25:17.633251",
+   "duration": 2.571431,
+   "end_time": "2020-04-18T18:11:06.657430",
   "environment_variables": {},
   "exception": null,
-   "input_path": "/tmp/pnqb5z0q/notebooks/process/download-covidtracking-data.ipynb",
+   "input_path": "/tmp/lvwjkfpt/notebooks/process/download-covidtracking-data.ipynb",
   "output_path": "runs/download-covidtracking-data.runs.ipynb",
   "parameters": {
-    "PAPERMILL_INPUT_PATH": "/tmp/pnqb5z0q/notebooks/process/download-covidtracking-data.ipynb",
+    "PAPERMILL_INPUT_PATH": "/tmp/lvwjkfpt/notebooks/process/download-covidtracking-data.ipynb",
    "PAPERMILL_OUTPUT_PATH": "runs/download-covidtracking-data.runs.ipynb",
    "out_folder": "data/covidtracking"
   },
-   "start_time": "2020-04-17T07:25:11.216335",
+   "start_time": "2020-04-18T18:11:04.085999",
   "version": "1.1.0"
  }
 },

 %% Cell type:code id: tags:

 ``` python
 import requests
 import os
 import pandas as pd
 ```

 %% Cell type:code id: tags:parameters

 ``` python
 out_folder = "../data/covidtracking/"
 PAPERMILL_OUTPUT_PATH = None
 ```

 %% Cell type:code id: tags:injected-parameters

 ``` python
 # Parameters
-PAPERMILL_INPUT_PATH = "/tmp/pnqb5z0q/notebooks/process/download-covidtracking-data.ipynb"
+PAPERMILL_INPUT_PATH = "/tmp/lvwjkfpt/notebooks/process/download-covidtracking-data.ipynb"
 PAPERMILL_OUTPUT_PATH = "runs/download-covidtracking-data.runs.ipynb"
 out_folder = "data/covidtracking"
 ```

 %% Cell type:markdown id: tags:

 # Download state metadata

 Download a dataset of URLs for data for each US state and several territories. See [Google Doc](https://docs.google.com/spreadsheets/d/18oVRrHj3c183mHmq3m89_163yuYltLNlOmPerQ18E8w/htmlview?sle=true).

 %% Cell type:code id: tags:

 ``` python
 url = 'http://covidtracking.com/api/states/info'
 r = requests.get(url, allow_redirects=True)
 states_metadata_json = r.content
 ```

 %% Cell type:code id: tags:

 ``` python
 # save the result
 if PAPERMILL_OUTPUT_PATH:
    out_path = os.path.join(out_folder, 'states-metadata.json')
    with open(out_path, 'wb') as f:
        f.write(states_metadata_json)
 ```

 %% Cell type:code id: tags:

 ``` python
 metadata_df = pd.read_json(states_metadata_json)
 print(len(metadata_df), "states and territories have metadata")
 metadata_df.head(2)
 ```

 %% Output

    56 states and territories have metadata

      state                                     covid19SiteOld  \
    0    AK  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...
    1    AL  http://www.alabamapublichealth.gov/infectiousd...
    
                                             covid19Site  \
    0  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...
    1  https://alpublichealth.maps.arcgis.com/apps/op...
    
                                    covid19SiteSecondary          twitter  \
    0  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...     @Alaska_DHSS
    1                                               None  @alpublichealth
    
            pui    pum                                              notes  fips  \
    0  All data  False  Total tests are taken from the annotations on ...     2
    1   No data  False  Negatives = (Totals - Positives) \nPositives o...     1
    
          name
    0   Alaska
    1  Alabama

 %% Cell type:markdown id: tags:

 # Download daily state data

 %% Cell type:code id: tags:

 ``` python
 url = 'https://covidtracking.com/api/states/daily'
 r = requests.get(url, allow_redirects=True)
 states_daily_json = r.content
 ```

 %% Cell type:code id: tags:

 ``` python
 # save the result
 if PAPERMILL_OUTPUT_PATH:
    out_path = os.path.join(out_folder, 'states-daily.json')
    with open(out_path, 'wb') as f:
        f.write(states_daily_json)
 ```

 %% Cell type:code id: tags:

 ``` python
 data_df = pd.read_json(states_daily_json)
 print(len(data_df), "data points")
 data_df.head(2)
 ```

 %% Output

-    2340 data points
+    2396 data points

           date state  positive  negative  pending  hospitalizedCurrently  \
-    0  20200416    AK     300.0    8435.0      NaN                    NaN
-    1  20200416    AL    4345.0   32046.0      NaN                    NaN
+    0  20200417    AK     309.0    9141.0      NaN                    NaN
+    1  20200417    AL    4530.0   33318.0      NaN                    NaN
    
       hospitalizedCumulative  inIcuCurrently  inIcuCumulative  \
-    0                    35.0             NaN              NaN
-    1                   553.0             NaN            227.0
+    0                    36.0             NaN              NaN
+    1                   594.0             NaN            247.0
    
       onVentilatorCurrently  ...  hospitalized    total totalTestResults  \
-    0                    NaN  ...          35.0   8735.0           8735.0
-    1                    NaN  ...         553.0  36391.0          36391.0
+    0                    NaN  ...          36.0   9450.0           9450.0
+    1                    NaN  ...         594.0  37848.0          37848.0
    
        posNeg  fips  deathIncrease  hospitalizedIncrease  negativeIncrease  \
-    0   8735.0     2            0.0                   1.0              64.0
-    1  36391.0     1           12.0                  28.0            2082.0
+    0   9450.0     2            0.0                   1.0             706.0
+    1  37848.0     1           11.0                  41.0            1272.0
    
       positiveIncrease  totalTestResultsIncrease
-    0               7.0                      71.0
-    1             232.0                    2314.0
+    0               9.0                     715.0
+    1             185.0                    1457.0
    
    [2 rows x 25 columns]