renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json

2d549e97 · CR (covid cron) · renku 0.9.1 · 67f0585c · 2d549e97 · 2d549e97
Commit 2d549e97 authored 4 years ago by CR (covid cron) Committed by renku 0.9.1 4 years ago
--- a/.renku/workflow/f2dbb09e70314ed4b235c8bf9cdd94a1.cwl
+++ b/.renku/workflow/f2dbb09e70314ed4b235c8bf9cdd94a1.cwl
+class: Workflow
+cwlVersion: v1.0
+hints: []
+inputs:
+  input_1:
+    default: states-daily.json
+    streamable: false
+    type: string
+  input_2:
+    default: out_folder
+    streamable: false
+    type: string
+  input_3:
+    default: data/covidtracking
+    streamable: false
+    type: string
+  input_4:
+    default:
+      class: File
+      path: ../../notebooks/process/download-covidtracking-data.ipynb
+    streamable: false
+    type: File
+  input_5:
+    default: runs/download-covidtracking-data.runs.ipynb
+    streamable: false
+    type: string
+  input_6:
+    default: states-metadata.json
+    streamable: false
+    type: string
+outputs:
+  output_0:
+    outputSource: step_2/output_0
+    streamable: false
+    type: File
+  output_1:
+    outputSource: step_2/output_1
+    streamable: false
+    type: Directory
+requirements: []
+steps:
+  step_1:
+    in:
+      filename: input_1
+      input_directory: step_2/output_1
+    out:
+    - output_file
+    run:
+      arguments: []
+      baseCommand:
+      - 'true'
+      class: CommandLineTool
+      cwlVersion: v1.0
+      hints: []
+      inputs:
+        filename:
+          default: states-daily.json
+          streamable: false
+          type: string
+        input_directory:
+          streamable: false
+          type: Directory
+      outputs:
+        output_file:
+          outputBinding:
+            glob: $(inputs.filename)
+          streamable: false
+          type: File
+      permanentFailCodes: []
+      requirements:
+      - &id001
+        class: InlineJavascriptRequirement
+      - &id002
+        class: InitialWorkDirRequirement
+        listing: $(inputs.input_directory.listing)
+      successCodes: []
+      temporaryFailCodes: []
+  step_2:
+    in:
+      input_1: input_2
+      input_2: input_3
+      input_3: input_4
+      input_4: input_5
+    out:
+    - output_1
+    - output_0
+    run: ff5f474d38144c5d882c0036bd1059c2_papermill.cwl
+  step_3:
+    in:
+      filename: input_6
+      input_directory: step_2/output_1
+    out:
+    - output_file
+    run:
+      arguments: []
+      baseCommand:
+      - 'true'
+      class: CommandLineTool
+      cwlVersion: v1.0
+      hints: []
+      inputs:
+        filename:
+          default: states-metadata.json
+          streamable: false
+          type: string
+        input_directory:
+          streamable: false
+          type: Directory
+      outputs:
+        output_file:
+          outputBinding:
+            glob: $(inputs.filename)
+          streamable: false
+          type: File
+      permanentFailCodes: []
+      requirements:
+      - *id001
+      - *id002
+      successCodes: []
+      temporaryFailCodes: []
--- a/data/covidtracking/states-daily.json
+++ b/data/covidtracking/states-daily.json
--- a/data/covidtracking/states-metadata.json
+++ b/data/covidtracking/states-metadata.json
--- a/runs/download-covidtracking-data.runs.ipynb
+++ b/runs/download-covidtracking-data.runs.ipynb
@@ -5,10 +5,10 @@
   "execution_count": 1,
   "metadata": {
    "papermill": {
-     "duration": 2.276469,
-     "end_time": "2020-03-23T08:22:22.453561",
+     "duration": 1.484962,
+     "end_time": "2020-03-24T11:08:51.882283",
     "exception": false,
-     "start_time": "2020-03-23T08:22:20.177092",
+     "start_time": "2020-03-24T11:08:50.397321",
     "status": "completed"
    },
    "tags": []
@@ -25,10 +25,10 @@
   "execution_count": 2,
   "metadata": {
    "papermill": {
-     "duration": 0.020852,
-     "end_time": "2020-03-23T08:22:22.488968",
+     "duration": 0.034224,
+     "end_time": "2020-03-24T11:08:51.927200",
     "exception": false,
-     "start_time": "2020-03-23T08:22:22.468116",
+     "start_time": "2020-03-24T11:08:51.892976",
     "status": "completed"
    },
    "tags": [
@@ -46,10 +46,10 @@
   "execution_count": 3,
   "metadata": {
    "papermill": {
-     "duration": 0.02465,
-     "end_time": "2020-03-23T08:22:22.522182",
+     "duration": 0.024298,
+     "end_time": "2020-03-24T11:08:51.965529",
     "exception": false,
-     "start_time": "2020-03-23T08:22:22.497532",
+     "start_time": "2020-03-24T11:08:51.941231",
     "status": "completed"
    },
    "tags": [
@@ -59,7 +59,7 @@
   "outputs": [],
   "source": [
    "# Parameters\n",
-    "PAPERMILL_INPUT_PATH = \"/tmp/iocfjzox/notebooks/process/download-covidtracking-data.ipynb\"\n",
+    "PAPERMILL_INPUT_PATH = \"/tmp/kq4860jy/notebooks/process/download-covidtracking-data.ipynb\"\n",
    "PAPERMILL_OUTPUT_PATH = \"runs/download-covidtracking-data.runs.ipynb\"\n",
    "out_folder = \"data/covidtracking\"\n"
   ]
@@ -68,10 +68,10 @@
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
-     "duration": 0.005919,
-     "end_time": "2020-03-23T08:22:22.536991",
+     "duration": 0.007354,
+     "end_time": "2020-03-24T11:08:51.981277",
     "exception": false,
-     "start_time": "2020-03-23T08:22:22.531072",
+     "start_time": "2020-03-24T11:08:51.973923",
     "status": "completed"
    },
    "tags": []
@@ -87,10 +87,10 @@
   "execution_count": 4,
   "metadata": {
    "papermill": {
-     "duration": 0.683247,
-     "end_time": "2020-03-23T08:22:23.226480",
+     "duration": 1.994092,
+     "end_time": "2020-03-24T11:08:53.981279",
     "exception": false,
-     "start_time": "2020-03-23T08:22:22.543233",
+     "start_time": "2020-03-24T11:08:51.987187",
     "status": "completed"
    },
    "tags": []
@@ -107,10 +107,10 @@
   "execution_count": 5,
   "metadata": {
    "papermill": {
-     "duration": 0.02999,
-     "end_time": "2020-03-23T08:22:23.270771",
+     "duration": 0.028446,
+     "end_time": "2020-03-24T11:08:54.020108",
     "exception": false,
-     "start_time": "2020-03-23T08:22:23.240781",
+     "start_time": "2020-03-24T11:08:53.991662",
     "status": "completed"
    },
    "tags": []
@@ -129,10 +129,10 @@
   "execution_count": 6,
   "metadata": {
    "papermill": {
-     "duration": 0.075074,
-     "end_time": "2020-03-23T08:22:23.356315",
+     "duration": 0.080991,
+     "end_time": "2020-03-24T11:08:54.110615",
     "exception": false,
-     "start_time": "2020-03-23T08:22:23.281241",
+     "start_time": "2020-03-24T11:08:54.029624",
     "status": "completed"
    },
    "tags": []
@@ -187,7 +187,7 @@
       "      <td>@Alaska_DHSS</td>\n",
       "      <td>All data</td>\n",
       "      <td>False</td>\n",
-       "      <td>Unclear if their reported number means \"person...</td>\n",
+       "      <td>We count the reported number as \"persons teste...</td>\n",
       "      <td>Alaska</td>\n",
       "    </tr>\n",
       "    <tr>\n",
@@ -220,7 +220,7 @@
       "1                                               None  @alpublichealth   \n",
       "\n",
       "        pui    pum                                              notes     name  \n",
-       "0  All data  False  Unclear if their reported number means \"person...   Alaska  \n",
+       "0  All data  False  We count the reported number as \"persons teste...   Alaska  \n",
       "1   No data  False  Last negative count from 3/16. Last update tim...  Alabama  "
      ]
     },
@@ -239,10 +239,10 @@
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
-     "duration": 0.008205,
-     "end_time": "2020-03-23T08:22:23.377337",
+     "duration": 0.010469,
+     "end_time": "2020-03-24T11:08:54.132051",
     "exception": false,
-     "start_time": "2020-03-23T08:22:23.369132",
+     "start_time": "2020-03-24T11:08:54.121582",
     "status": "completed"
    },
    "tags": []
@@ -256,10 +256,10 @@
   "execution_count": 7,
   "metadata": {
    "papermill": {
-     "duration": 0.216238,
-     "end_time": "2020-03-23T08:22:23.599867",
+     "duration": 0.162493,
+     "end_time": "2020-03-24T11:08:54.304650",
     "exception": false,
-     "start_time": "2020-03-23T08:22:23.383629",
+     "start_time": "2020-03-24T11:08:54.142157",
     "status": "completed"
    },
    "tags": []
@@ -276,10 +276,10 @@
   "execution_count": 8,
   "metadata": {
    "papermill": {
-     "duration": 0.030954,
-     "end_time": "2020-03-23T08:22:23.651293",
+     "duration": 0.02792,
+     "end_time": "2020-03-24T11:08:54.344357",
     "exception": false,
-     "start_time": "2020-03-23T08:22:23.620339",
+     "start_time": "2020-03-24T11:08:54.316437",
     "status": "completed"
    },
    "tags": []
@@ -298,10 +298,10 @@
   "execution_count": 9,
   "metadata": {
    "papermill": {
-     "duration": 0.066839,
-     "end_time": "2020-03-23T08:22:23.732970",
+     "duration": 0.06445,
+     "end_time": "2020-03-24T11:08:54.419822",
     "exception": false,
-     "start_time": "2020-03-23T08:22:23.666131",
+     "start_time": "2020-03-24T11:08:54.355372",
     "status": "completed"
    },
    "tags": []
@@ -311,7 +311,7 @@
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "925 data points\n"
+      "981 data points\n"
     ]
    },
    {
@@ -349,7 +349,7 @@
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
-       "      <td>20200322</td>\n",
+       "      <td>20200323</td>\n",
       "      <td>AK</td>\n",
       "      <td>22.0</td>\n",
       "      <td>946.0</td>\n",
@@ -357,19 +357,19 @@
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>968</td>\n",
-       "      <td>2020-03-22T20:00:00Z</td>\n",
+       "      <td>2020-03-23T20:00:00Z</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
-       "      <td>20200322</td>\n",
+       "      <td>20200323</td>\n",
       "      <td>AL</td>\n",
-       "      <td>138.0</td>\n",
-       "      <td>1464.0</td>\n",
+       "      <td>167.0</td>\n",
+       "      <td>1665.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
-       "      <td>1602</td>\n",
-       "      <td>2020-03-22T20:00:00Z</td>\n",
+       "      <td>1832</td>\n",
+       "      <td>2020-03-23T20:00:00Z</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
@@ -377,12 +377,12 @@
      ],
      "text/plain": [
       "       date state  positive  negative  pending  hospitalized  death  total  \\\n",
-       "0  20200322    AK      22.0     946.0      NaN           0.0    NaN    968   \n",
-       "1  20200322    AL     138.0    1464.0      NaN           NaN    0.0   1602   \n",
+       "0  20200323    AK      22.0     946.0      NaN           0.0    NaN    968   \n",
+       "1  20200323    AL     167.0    1665.0      NaN           NaN    0.0   1832   \n",
       "\n",
       "            dateChecked  \n",
-       "0  2020-03-22T20:00:00Z  \n",
-       "1  2020-03-22T20:00:00Z  "
+       "0  2020-03-23T20:00:00Z  \n",
+       "1  2020-03-23T20:00:00Z  "
      ]
     },
     "execution_count": 9,
@@ -416,18 +416,18 @@
   "version": "3.7.3"
  },
  "papermill": {
-   "duration": 4.883343,
-   "end_time": "2020-03-23T08:22:24.072161",
+   "duration": 5.713541,
+   "end_time": "2020-03-24T11:08:54.762805",
   "environment_variables": {},
   "exception": null,
-   "input_path": "/tmp/iocfjzox/notebooks/process/download-covidtracking-data.ipynb",
+   "input_path": "/tmp/kq4860jy/notebooks/process/download-covidtracking-data.ipynb",
   "output_path": "runs/download-covidtracking-data.runs.ipynb",
   "parameters": {
-    "PAPERMILL_INPUT_PATH": "/tmp/iocfjzox/notebooks/process/download-covidtracking-data.ipynb",
+    "PAPERMILL_INPUT_PATH": "/tmp/kq4860jy/notebooks/process/download-covidtracking-data.ipynb",
    "PAPERMILL_OUTPUT_PATH": "runs/download-covidtracking-data.runs.ipynb",
    "out_folder": "data/covidtracking"
   },
-   "start_time": "2020-03-23T08:22:19.188818",
+   "start_time": "2020-03-24T11:08:49.049264",
   "version": "1.1.0"
  }
 },

 %% Cell type:code id: tags:

 ``` python
 import requests
 import os
 import pandas as pd
 ```

 %% Cell type:code id: tags:parameters

 ``` python
 out_folder = "../data/covidtracking/"
 PAPERMILL_OUTPUT_PATH = None
 ```

 %% Cell type:code id: tags:injected-parameters

 ``` python
 # Parameters
-PAPERMILL_INPUT_PATH = "/tmp/iocfjzox/notebooks/process/download-covidtracking-data.ipynb"
+PAPERMILL_INPUT_PATH = "/tmp/kq4860jy/notebooks/process/download-covidtracking-data.ipynb"
 PAPERMILL_OUTPUT_PATH = "runs/download-covidtracking-data.runs.ipynb"
 out_folder = "data/covidtracking"
 ```

 %% Cell type:markdown id: tags:

 # Download state metadata

 Download a dataset of URLs for data for each US state and several territories. See [Google Doc](https://docs.google.com/spreadsheets/d/18oVRrHj3c183mHmq3m89_163yuYltLNlOmPerQ18E8w/htmlview?sle=true).

 %% Cell type:code id: tags:

 ``` python
 url = 'http://covidtracking.com/api/states/info'
 r = requests.get(url, allow_redirects=True)
 states_metadata_json = r.content
 ```

 %% Cell type:code id: tags:

 ``` python
 # save the result
 if PAPERMILL_OUTPUT_PATH:
    out_path = os.path.join(out_folder, 'states-metadata.json')
    with open(out_path, 'wb') as f:
        f.write(states_metadata_json)
 ```

 %% Cell type:code id: tags:

 ``` python
 metadata_df = pd.read_json(states_metadata_json)
 print(len(metadata_df), "states and territories have metadata")
 metadata_df.head(2)
 ```

 %% Output

    56 states and territories have metadata

      state                                     covid19SiteOld  \
    0    AK  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...
    1    AL  http://www.alabamapublichealth.gov/infectiousd...
    
                                             covid19Site  \
    0  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...
    1  https://alpublichealth.maps.arcgis.com/apps/op...
    
                                    covid19SiteSecondary          twitter  \
    0  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...     @Alaska_DHSS
    1                                               None  @alpublichealth
    
            pui    pum                                              notes     name
-    0  All data  False  Unclear if their reported number means "person...   Alaska
+    0  All data  False  We count the reported number as "persons teste...   Alaska
    1   No data  False  Last negative count from 3/16. Last update tim...  Alabama

 %% Cell type:markdown id: tags:

 # Download daily state data

 %% Cell type:code id: tags:

 ``` python
 url = 'https://covidtracking.com/api/states/daily'
 r = requests.get(url, allow_redirects=True)
 states_daily_json = r.content
 ```

 %% Cell type:code id: tags:

 ``` python
 # save the result
 if PAPERMILL_OUTPUT_PATH:
    out_path = os.path.join(out_folder, 'states-daily.json')
    with open(out_path, 'wb') as f:
        f.write(states_daily_json)
 ```

 %% Cell type:code id: tags:

 ``` python
 data_df = pd.read_json(states_daily_json)
 print(len(data_df), "data points")
 data_df.head(2)
 ```

 %% Output

-    925 data points
+    981 data points

           date state  positive  negative  pending  hospitalized  death  total  \
-    0  20200322    AK      22.0     946.0      NaN           0.0    NaN    968
-    1  20200322    AL     138.0    1464.0      NaN           NaN    0.0   1602
+    0  20200323    AK      22.0     946.0      NaN           0.0    NaN    968
+    1  20200323    AL     167.0    1665.0      NaN           NaN    0.0   1832
    
                dateChecked
-    0  2020-03-22T20:00:00Z
-    1  2020-03-22T20:00:00Z
+    0  2020-03-23T20:00:00Z
+    1  2020-03-23T20:00:00Z