renku run papermill -p out_folder ./data/covidtracking/ --inject-paths...

renku run papermill -p out_folder ./data/covidtracking/ --inject-paths notebooks/process/download-covidtracking-data.ipynb runs/download-covidtracking-data.runs.ipynb

renku run papermill -p out_folder ./data/covidtracking/ --inject-paths...
renku run papermill -p out_folder ./data/covidtracking/ --inject-paths notebooks/process/download-covidtracking-data.ipynb runs/download-covidtracking-data.runs.ipynb
ccc37edf · Chandrasekhar Ramakrishnan · renku 0.9.1 · 56912200 · ccc37edf · ccc37edf
Commit ccc37edf authored 4 years ago by Chandrasekhar Ramakrishnan Committed by renku 0.9.1 4 years ago
--- a/.renku/workflow/a17d560c41a54f5aa307ce5f3c5effe5_papermill.cwl
+++ b/.renku/workflow/a17d560c41a54f5aa307ce5f3c5effe5_papermill.cwl
+arguments: []
+baseCommand:
+- papermill
+class: CommandLineTool
+cwlVersion: v1.0
+hints: []
+inputs:
+  input_1:
+    default: out_folder
+    inputBinding:
+      position: 1
+      prefix: -p
+      separate: true
+      shellQuote: true
+    streamable: false
+    type: string
+  input_2:
+    default: data/covidtracking
+    inputBinding:
+      position: 2
+      separate: true
+      shellQuote: true
+    streamable: false
+    type: string
+  input_3:
+    default:
+      class: File
+      path: ../../notebooks/process/download-covidtracking-data.ipynb
+    inputBinding:
+      position: 3
+      prefix: --inject-paths
+      separate: true
+      shellQuote: true
+    streamable: false
+    type: File
+  input_4:
+    default: runs/download-covidtracking-data.runs.ipynb
+    inputBinding:
+      position: 4
+      separate: true
+      shellQuote: true
+    streamable: false
+    type: string
+outputs:
+  output_0:
+    outputBinding:
+      glob: $(inputs.input_4)
+    streamable: false
+    type: File
+  output_1:
+    outputBinding:
+      glob: $(inputs.input_2)
+    streamable: false
+    type: Directory
+permanentFailCodes: []
+requirements:
+- class: InlineJavascriptRequirement
+- class: InitialWorkDirRequirement
+  listing:
+  - entry: '$({"listing": [], "class": "Directory"})'
+    entryname: runs
+    writable: true
+  - entry: '$({"listing": [], "class": "Directory"})'
+    entryname: data/covidtracking
+    writable: true
+  - entry: $(inputs.input_3)
+    entryname: notebooks/process/download-covidtracking-data.ipynb
+    writable: false
+successCodes: []
+temporaryFailCodes: []
--- a/data/covidtracking/states-daily.json
+++ b/data/covidtracking/states-daily.json
--- a/data/covidtracking/states-metadata.json
+++ b/data/covidtracking/states-metadata.json
--- a/runs/download-covidtracking-data.runs.ipynb
+++ b/runs/download-covidtracking-data.runs.ipynb
@@ -5,10 +5,10 @@
   "execution_count": 1,
   "metadata": {
    "papermill": {
-     "duration": 0.564716,
+     "duration": 0.506363,
-     "end_time": "2020-03-25T08:22:56.714459",
+     "end_time": "2020-03-26T12:38:33.422061",
     "exception": false,
-     "start_time": "2020-03-25T08:22:56.149743",
+     "start_time": "2020-03-26T12:38:32.915698",
     "status": "completed"
    },
    "tags": []
@@ -25,10 +25,10 @@
   "execution_count": 2,
   "metadata": {
    "papermill": {
-     "duration": 0.019507,
+     "duration": 0.020958,
-     "end_time": "2020-03-25T08:22:56.745116",
+     "end_time": "2020-03-26T12:38:33.458041",
     "exception": false,
-     "start_time": "2020-03-25T08:22:56.725609",
+     "start_time": "2020-03-26T12:38:33.437083",
     "status": "completed"
    },
    "tags": [
@@ -46,10 +46,10 @@
   "execution_count": 3,
   "metadata": {
    "papermill": {
-     "duration": 0.018572,
+     "duration": 0.023542,
-     "end_time": "2020-03-25T08:22:56.773076",
+     "end_time": "2020-03-26T12:38:33.490779",
     "exception": false,
-     "start_time": "2020-03-25T08:22:56.754504",
+     "start_time": "2020-03-26T12:38:33.467237",
     "status": "completed"
    },
    "tags": [
@@ -59,19 +59,19 @@
   "outputs": [],
   "source": [
    "# Parameters\n",
-    "PAPERMILL_INPUT_PATH = \"/tmp/e18fw9c9/notebooks/process/download-covidtracking-data.ipynb\"\n",
+    "PAPERMILL_INPUT_PATH = \"notebooks/process/download-covidtracking-data.ipynb\"\n",
    "PAPERMILL_OUTPUT_PATH = \"runs/download-covidtracking-data.runs.ipynb\"\n",
-    "out_folder = \"data/covidtracking\"\n"
+    "out_folder = \"./data/covidtracking/\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
-     "duration": 0.007795,
+     "duration": 0.007637,
-     "end_time": "2020-03-25T08:22:56.790849",
+     "end_time": "2020-03-26T12:38:33.508933",
     "exception": false,
-     "start_time": "2020-03-25T08:22:56.783054",
+     "start_time": "2020-03-26T12:38:33.501296",
     "status": "completed"
    },
    "tags": []
@@ -87,10 +87,10 @@
   "execution_count": 4,
   "metadata": {
    "papermill": {
-     "duration": 1.020789,
+     "duration": 0.211295,
-     "end_time": "2020-03-25T08:22:57.818863",
+     "end_time": "2020-03-26T12:38:33.726495",
     "exception": false,
-     "start_time": "2020-03-25T08:22:56.798074",
+     "start_time": "2020-03-26T12:38:33.515200",
     "status": "completed"
    },
    "tags": []
@@ -107,10 +107,10 @@
   "execution_count": 5,
   "metadata": {
    "papermill": {
-     "duration": 0.022913,
+     "duration": 0.025576,
-     "end_time": "2020-03-25T08:22:57.852810",
+     "end_time": "2020-03-26T12:38:33.768948",
     "exception": false,
-     "start_time": "2020-03-25T08:22:57.829897",
+     "start_time": "2020-03-26T12:38:33.743372",
     "status": "completed"
    },
    "tags": []
@@ -129,10 +129,10 @@
   "execution_count": 6,
   "metadata": {
    "papermill": {
-     "duration": 0.066514,
+     "duration": 0.073344,
-     "end_time": "2020-03-25T08:22:57.927763",
+     "end_time": "2020-03-26T12:38:33.849972",
     "exception": false,
-     "start_time": "2020-03-25T08:22:57.861249",
+     "start_time": "2020-03-26T12:38:33.776628",
     "status": "completed"
    },
    "tags": []
@@ -174,6 +174,7 @@
       "      <th>pui</th>\n",
       "      <th>pum</th>\n",
       "      <th>notes</th>\n",
+       "      <th>fips</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
@@ -188,6 +189,7 @@
       "      <td>All data</td>\n",
       "      <td>False</td>\n",
       "      <td>We count the reported number as \"persons teste...</td>\n",
+       "      <td>2</td>\n",
       "      <td>Alaska</td>\n",
       "    </tr>\n",
       "    <tr>\n",
@@ -200,6 +202,7 @@
       "      <td>No data</td>\n",
       "      <td>False</td>\n",
       "      <td>Last update time taken from [main page](http:/...</td>\n",
+       "      <td>1</td>\n",
       "      <td>Alabama</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
@@ -219,9 +222,13 @@
       "0  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...     @Alaska_DHSS   \n",
       "1                                               None  @alpublichealth   \n",
       "\n",
-       "        pui    pum                                              notes     name  \n",
+       "        pui    pum                                              notes  fips  \\\n",
-       "0  All data  False  We count the reported number as \"persons teste...   Alaska  \n",
+       "0  All data  False  We count the reported number as \"persons teste...     2   \n",
-       "1   No data  False  Last update time taken from [main page](http:/...  Alabama  "
+       "1   No data  False  Last update time taken from [main page](http:/...     1   \n",
+       "\n",
+       "      name  \n",
+       "0   Alaska  \n",
+       "1  Alabama  "
      ]
     },
     "execution_count": 6,
@@ -239,10 +246,10 @@
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
-     "duration": 0.008089,
+     "duration": 0.007572,
-     "end_time": "2020-03-25T08:22:57.947587",
+     "end_time": "2020-03-26T12:38:33.873621",
     "exception": false,
-     "start_time": "2020-03-25T08:22:57.939498",
+     "start_time": "2020-03-26T12:38:33.866049",
     "status": "completed"
    },
    "tags": []
@@ -256,10 +263,10 @@
   "execution_count": 7,
   "metadata": {
    "papermill": {
-     "duration": 0.184993,
+     "duration": 0.178588,
-     "end_time": "2020-03-25T08:22:58.140940",
+     "end_time": "2020-03-26T12:38:34.058549",
     "exception": false,
-     "start_time": "2020-03-25T08:22:57.955947",
+     "start_time": "2020-03-26T12:38:33.879961",
     "status": "completed"
    },
    "tags": []
@@ -276,10 +283,10 @@
   "execution_count": 8,
   "metadata": {
    "papermill": {
-     "duration": 0.022977,
+     "duration": 0.027495,
-     "end_time": "2020-03-25T08:22:58.177014",
+     "end_time": "2020-03-26T12:38:34.102339",
     "exception": false,
-     "start_time": "2020-03-25T08:22:58.154037",
+     "start_time": "2020-03-26T12:38:34.074844",
     "status": "completed"
    },
    "tags": []
@@ -298,10 +305,10 @@
   "execution_count": 9,
   "metadata": {
    "papermill": {
-     "duration": 0.060602,
+     "duration": 0.076898,
-     "end_time": "2020-03-25T08:22:58.251228",
+     "end_time": "2020-03-26T12:38:34.188602",
     "exception": false,
-     "start_time": "2020-03-25T08:22:58.190626",
+     "start_time": "2020-03-26T12:38:34.111704",
     "status": "completed"
    },
    "tags": []
@@ -311,7 +318,7 @@
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "1037 data points\n"
+      "1093 data points\n"
     ]
    },
    {
@@ -344,32 +351,50 @@
       "      <th>death</th>\n",
       "      <th>total</th>\n",
       "      <th>dateChecked</th>\n",
+       "      <th>totalTestResults</th>\n",
+       "      <th>deathIncrease</th>\n",
+       "      <th>hospitalizedIncrease</th>\n",
+       "      <th>negativeIncrease</th>\n",
+       "      <th>positiveIncrease</th>\n",
+       "      <th>totalTestResultsIncrease</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
-       "      <td>20200324</td>\n",
+       "      <td>20200325</td>\n",
       "      <td>AK</td>\n",
-       "      <td>36.0</td>\n",
+       "      <td>42.0</td>\n",
-       "      <td>986.0</td>\n",
+       "      <td>1649.0</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
-       "      <td>1022</td>\n",
+       "      <td>1.0</td>\n",
-       "      <td>2020-03-24T20:00:00Z</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>1691</td>\n",
+       "      <td>2020-03-25T20:00:00Z</td>\n",
+       "      <td>1691</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>663.0</td>\n",
+       "      <td>6.0</td>\n",
+       "      <td>669.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
-       "      <td>20200324</td>\n",
+       "      <td>20200325</td>\n",
       "      <td>AL</td>\n",
-       "      <td>215.0</td>\n",
+       "      <td>283.0</td>\n",
-       "      <td>2106.0</td>\n",
+       "      <td>2529.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
-       "      <td>2321</td>\n",
+       "      <td>2812</td>\n",
-       "      <td>2020-03-24T20:00:00Z</td>\n",
+       "      <td>2020-03-25T20:00:00Z</td>\n",
+       "      <td>2812</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>423.0</td>\n",
+       "      <td>68.0</td>\n",
+       "      <td>491.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
@@ -377,12 +402,20 @@
      ],
      "text/plain": [
       "       date state  positive  negative  pending  hospitalized  death  total  \\\n",
-       "0  20200324    AK      36.0     986.0      NaN           0.0    NaN   1022   \n",
+       "0  20200325    AK      42.0    1649.0      NaN           1.0    1.0   1691   \n",
-       "1  20200324    AL     215.0    2106.0      NaN           NaN    0.0   2321   \n",
+       "1  20200325    AL     283.0    2529.0      NaN           NaN    0.0   2812   \n",
+       "\n",
+       "            dateChecked  totalTestResults  deathIncrease  \\\n",
+       "0  2020-03-25T20:00:00Z              1691            1.0   \n",
+       "1  2020-03-25T20:00:00Z              2812            0.0   \n",
+       "\n",
+       "   hospitalizedIncrease  negativeIncrease  positiveIncrease  \\\n",
+       "0                   1.0             663.0               6.0   \n",
+       "1                   0.0             423.0              68.0   \n",
       "\n",
-       "            dateChecked  \n",
+       "   totalTestResultsIncrease  \n",
-       "0  2020-03-24T20:00:00Z  \n",
+       "0                     669.0  \n",
-       "1  2020-03-24T20:00:00Z  "
+       "1                     491.0  "
      ]
     },
     "execution_count": 9,
@@ -416,18 +449,18 @@
   "version": "3.7.3"
  },
  "papermill": {
-   "duration": 3.460914,
+   "duration": 2.475212,
-   "end_time": "2020-03-25T08:22:58.575452",
+   "end_time": "2020-03-26T12:38:34.517904",
   "environment_variables": {},
   "exception": null,
-   "input_path": "/tmp/e18fw9c9/notebooks/process/download-covidtracking-data.ipynb",
+   "input_path": "notebooks/process/download-covidtracking-data.ipynb",
   "output_path": "runs/download-covidtracking-data.runs.ipynb",
   "parameters": {
-    "PAPERMILL_INPUT_PATH": "/tmp/e18fw9c9/notebooks/process/download-covidtracking-data.ipynb",
+    "PAPERMILL_INPUT_PATH": "notebooks/process/download-covidtracking-data.ipynb",
    "PAPERMILL_OUTPUT_PATH": "runs/download-covidtracking-data.runs.ipynb",
-    "out_folder": "data/covidtracking"
+    "out_folder": "./data/covidtracking/"
   },
-   "start_time": "2020-03-25T08:22:55.114538",
+   "start_time": "2020-03-26T12:38:32.042692",
   "version": "1.1.0"
  }
 },

 %% Cell type:code id: tags:
 ``` python
 import requests
 import os
 import pandas as pd
 ```
 %% Cell type:code id: tags:parameters
 ``` python
 out_folder = "../data/covidtracking/"
 PAPERMILL_OUTPUT_PATH = None
 ```
 %% Cell type:code id: tags:injected-parameters
 ``` python
 # Parameters
-PAPERMILL_INPUT_PATH = "/tmp/e18fw9c9/notebooks/process/download-covidtracking-data.ipynb"
+PAPERMILL_INPUT_PATH = "notebooks/process/download-covidtracking-data.ipynb"
 PAPERMILL_OUTPUT_PATH = "runs/download-covidtracking-data.runs.ipynb"
-out_folder = "data/covidtracking"
+out_folder = "./data/covidtracking/"
 ```
 %% Cell type:markdown id: tags:
 # Download state metadata
 Download a dataset of URLs for data for each US state and several territories. See [Google Doc](https://docs.google.com/spreadsheets/d/18oVRrHj3c183mHmq3m89_163yuYltLNlOmPerQ18E8w/htmlview?sle=true).
 %% Cell type:code id: tags:
 ``` python
 url = 'http://covidtracking.com/api/states/info'
 r = requests.get(url, allow_redirects=True)
 states_metadata_json = r.content
 ```
 %% Cell type:code id: tags:
 ``` python
 # save the result
 if PAPERMILL_OUTPUT_PATH:
    out_path = os.path.join(out_folder, 'states-metadata.json')
    with open(out_path, 'wb') as f:
        f.write(states_metadata_json)
 ```
 %% Cell type:code id: tags:
 ``` python
 metadata_df = pd.read_json(states_metadata_json)
 print(len(metadata_df), "states and territories have metadata")
 metadata_df.head(2)
 ```
 %% Output
    56 states and territories have metadata
      state                                     covid19SiteOld  \
    0    AK  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...
    1    AL  http://www.alabamapublichealth.gov/infectiousd...
                                             covid19Site  \
    0  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...
    1  https://alpublichealth.maps.arcgis.com/apps/op...
                                    covid19SiteSecondary          twitter  \
    0  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...     @Alaska_DHSS
    1                                               None  @alpublichealth
-            pui    pum                                              notes     name
+            pui    pum                                              notes  fips  \
-    0  All data  False  We count the reported number as "persons teste...   Alaska
+    0  All data  False  We count the reported number as "persons teste...     2
-    1   No data  False  Last update time taken from [main page](http:/...  Alabama
+    1   No data  False  Last update time taken from [main page](http:/...     1
+          name
+    0   Alaska
+    1  Alabama
 %% Cell type:markdown id: tags:
 # Download daily state data
 %% Cell type:code id: tags:
 ``` python
 url = 'https://covidtracking.com/api/states/daily'
 r = requests.get(url, allow_redirects=True)
 states_daily_json = r.content
 ```
 %% Cell type:code id: tags:
 ``` python
 # save the result
 if PAPERMILL_OUTPUT_PATH:
    out_path = os.path.join(out_folder, 'states-daily.json')
    with open(out_path, 'wb') as f:
        f.write(states_daily_json)
 ```
 %% Cell type:code id: tags:
 ``` python
 data_df = pd.read_json(states_daily_json)
 print(len(data_df), "data points")
 data_df.head(2)
 ```
 %% Output
-    1037 data points
+    1093 data points
           date state  positive  negative  pending  hospitalized  death  total  \
-    0  20200324    AK      36.0     986.0      NaN           0.0    NaN   1022
+    0  20200325    AK      42.0    1649.0      NaN           1.0    1.0   1691
-    1  20200324    AL     215.0    2106.0      NaN           NaN    0.0   2321
+    1  20200325    AL     283.0    2529.0      NaN           NaN    0.0   2812
+                dateChecked  totalTestResults  deathIncrease  \
+    0  2020-03-25T20:00:00Z              1691            1.0
+    1  2020-03-25T20:00:00Z              2812            0.0
+       hospitalizedIncrease  negativeIncrease  positiveIncrease  \
+    0                   1.0             663.0               6.0
+    1                   0.0             423.0              68.0
-                dateChecked
+       totalTestResultsIncrease
-    0  2020-03-24T20:00:00Z
+    0                     669.0
-    1  2020-03-24T20:00:00Z
+    1                     491.0