renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json

56b0b3f4 · CI-bot · renku 0.10.4 · 2c65904f · 56b0b3f4 · 56b0b3f4
Commit 56b0b3f4 authored 4 years ago by CI-bot Committed by renku 0.10.4 4 years ago
--- a/.renku/workflow/8da137e532e245ab9b4eec02a551f756.cwl
+++ b/.renku/workflow/8da137e532e245ab9b4eec02a551f756.cwl
+class: Workflow
+cwlVersion: v1.0
+hints: []
+inputs:
+  input_1:
+    default: states-daily.json
+    streamable: false
+    type: string
+  input_2:
+    default: out_folder
+    streamable: false
+    type: string
+  input_3:
+    default: data/covidtracking
+    streamable: false
+    type: string
+  input_4:
+    default:
+      class: File
+      path: ../../notebooks/process/download-covidtracking-data.ipynb
+    streamable: false
+    type: File
+  input_5:
+    default: runs/download-covidtracking-data.runs.ipynb
+    streamable: false
+    type: string
+  input_6:
+    default: states-metadata.json
+    streamable: false
+    type: string
+outputs:
+  output_0:
+    outputSource: step_2/output_0
+    streamable: false
+    type: File
+  output_2:
+    outputSource: step_2/output_1
+    streamable: false
+    type: Directory
+requirements: []
+steps:
+  step_1:
+    in:
+      filename: input_1
+      input_directory: step_2/output_1
+    out:
+    - output_file
+    run:
+      arguments: []
+      baseCommand:
+      - 'true'
+      class: CommandLineTool
+      cwlVersion: v1.0
+      hints: []
+      inputs:
+        filename:
+          default: states-daily.json
+          streamable: false
+          type: string
+        input_directory:
+          streamable: false
+          type: Directory
+      outputs:
+        output_file:
+          outputBinding:
+            glob: $(inputs.filename)
+          streamable: false
+          type: File
+      permanentFailCodes: []
+      requirements:
+      - &id001
+        class: InlineJavascriptRequirement
+      - &id002
+        class: InitialWorkDirRequirement
+        listing: $(inputs.input_directory.listing)
+      successCodes: []
+      temporaryFailCodes: []
+  step_2:
+    in:
+      input_1: input_2
+      input_2: input_3
+      input_3: input_4
+      input_4: input_5
+    out:
+    - output_0
+    - output_1
+    run: a17d560c41a54f5aa307ce5f3c5effe5_papermill.cwl
+  step_3:
+    in:
+      filename: input_6
+      input_directory: step_2/output_1
+    out:
+    - output_file
+    run:
+      arguments: []
+      baseCommand:
+      - 'true'
+      class: CommandLineTool
+      cwlVersion: v1.0
+      hints: []
+      inputs:
+        filename:
+          default: states-metadata.json
+          streamable: false
+          type: string
+        input_directory:
+          streamable: false
+          type: Directory
+      outputs:
+        output_file:
+          outputBinding:
+            glob: $(inputs.filename)
+          streamable: false
+          type: File
+      permanentFailCodes: []
+      requirements:
+      - *id001
+      - *id002
+      successCodes: []
+      temporaryFailCodes: []
--- a/data/covidtracking/states-daily.json
+++ b/data/covidtracking/states-daily.json
--- a/data/covidtracking/states-metadata.json
+++ b/data/covidtracking/states-metadata.json
--- a/runs/download-covidtracking-data.runs.ipynb
+++ b/runs/download-covidtracking-data.runs.ipynb
@@ -5,10 +5,10 @@
   "execution_count": 1,
   "metadata": {
    "papermill": {
-     "duration": 4.046085,
-     "end_time": "2020-10-02T07:38:06.758857",
+     "duration": 5.253052,
+     "end_time": "2020-10-03T07:35:25.026066",
     "exception": false,
-     "start_time": "2020-10-02T07:38:02.712772",
+     "start_time": "2020-10-03T07:35:19.773014",
     "status": "completed"
    },
    "tags": []
@@ -25,10 +25,10 @@
   "execution_count": 2,
   "metadata": {
    "papermill": {
-     "duration": 0.025391,
-     "end_time": "2020-10-02T07:38:06.803555",
+     "duration": 0.022596,
+     "end_time": "2020-10-03T07:35:25.064552",
     "exception": false,
-     "start_time": "2020-10-02T07:38:06.778164",
+     "start_time": "2020-10-03T07:35:25.041956",
     "status": "completed"
    },
    "tags": [
@@ -46,10 +46,10 @@
   "execution_count": 3,
   "metadata": {
    "papermill": {
-     "duration": 0.02627,
-     "end_time": "2020-10-02T07:38:06.839437",
+     "duration": 0.020122,
+     "end_time": "2020-10-03T07:35:25.093667",
     "exception": false,
-     "start_time": "2020-10-02T07:38:06.813167",
+     "start_time": "2020-10-03T07:35:25.073545",
     "status": "completed"
    },
    "tags": [
@@ -59,7 +59,7 @@
   "outputs": [],
   "source": [
    "# Parameters\n",
-    "PAPERMILL_INPUT_PATH = \"/tmp/el6ip4bh/notebooks/process/download-covidtracking-data.ipynb\"\n",
+    "PAPERMILL_INPUT_PATH = \"/tmp/zy0hkj0c/notebooks/process/download-covidtracking-data.ipynb\"\n",
    "PAPERMILL_OUTPUT_PATH = \"runs/download-covidtracking-data.runs.ipynb\"\n",
    "out_folder = \"data/covidtracking\"\n"
   ]
@@ -68,10 +68,10 @@
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
-     "duration": 0.008354,
-     "end_time": "2020-10-02T07:38:06.861958",
+     "duration": 0.006778,
+     "end_time": "2020-10-03T07:35:25.108259",
     "exception": false,
-     "start_time": "2020-10-02T07:38:06.853604",
+     "start_time": "2020-10-03T07:35:25.101481",
     "status": "completed"
    },
    "tags": []
@@ -87,10 +87,10 @@
   "execution_count": 4,
   "metadata": {
    "papermill": {
-     "duration": 0.758127,
-     "end_time": "2020-10-02T07:38:07.627135",
+     "duration": 0.532189,
+     "end_time": "2020-10-03T07:35:25.645896",
     "exception": false,
-     "start_time": "2020-10-02T07:38:06.869008",
+     "start_time": "2020-10-03T07:35:25.113707",
     "status": "completed"
    },
    "tags": []
@@ -107,10 +107,10 @@
   "execution_count": 5,
   "metadata": {
    "papermill": {
-     "duration": 0.02875,
-     "end_time": "2020-10-02T07:38:07.669426",
+     "duration": 0.022618,
+     "end_time": "2020-10-03T07:35:25.681256",
     "exception": false,
-     "start_time": "2020-10-02T07:38:07.640676",
+     "start_time": "2020-10-03T07:35:25.658638",
     "status": "completed"
    },
    "tags": []
@@ -129,10 +129,10 @@
   "execution_count": 6,
   "metadata": {
    "papermill": {
-     "duration": 0.109723,
-     "end_time": "2020-10-02T07:38:07.793523",
+     "duration": 0.091493,
+     "end_time": "2020-10-03T07:35:25.780175",
     "exception": false,
-     "start_time": "2020-10-02T07:38:07.683800",
+     "start_time": "2020-10-03T07:35:25.688682",
     "status": "completed"
    },
    "tags": []
@@ -280,10 +280,10 @@
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
-     "duration": 0.007719,
-     "end_time": "2020-10-02T07:38:07.814061",
+     "duration": 0.006967,
+     "end_time": "2020-10-03T07:35:25.803168",
     "exception": false,
-     "start_time": "2020-10-02T07:38:07.806342",
+     "start_time": "2020-10-03T07:35:25.796201",
     "status": "completed"
    },
    "tags": []
@@ -297,10 +297,10 @@
   "execution_count": 7,
   "metadata": {
    "papermill": {
-     "duration": 26.639647,
-     "end_time": "2020-10-02T07:38:34.460276",
+     "duration": 26.252126,
+     "end_time": "2020-10-03T07:35:52.061529",
     "exception": false,
-     "start_time": "2020-10-02T07:38:07.820629",
+     "start_time": "2020-10-03T07:35:25.809403",
     "status": "completed"
    },
    "tags": []
@@ -317,10 +317,10 @@
   "execution_count": 8,
   "metadata": {
    "papermill": {
-     "duration": 0.045923,
-     "end_time": "2020-10-02T07:38:34.525196",
+     "duration": 0.06406,
+     "end_time": "2020-10-03T07:35:52.143430",
     "exception": false,
-     "start_time": "2020-10-02T07:38:34.479273",
+     "start_time": "2020-10-03T07:35:52.079370",
     "status": "completed"
    },
    "tags": []
@@ -339,10 +339,10 @@
   "execution_count": 9,
   "metadata": {
    "papermill": {
-     "duration": 0.702638,
-     "end_time": "2020-10-02T07:38:35.242651",
+     "duration": 0.575261,
+     "end_time": "2020-10-03T07:35:52.733021",
     "exception": false,
-     "start_time": "2020-10-02T07:38:34.540013",
+     "start_time": "2020-10-03T07:35:52.157760",
     "status": "completed"
    },
    "tags": []
@@ -352,7 +352,7 @@
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "11858 data points\n"
+      "11914 data points\n"
     ]
    },
    {
@@ -402,21 +402,21 @@
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
-       "      <td>20201001</td>\n",
+       "      <td>20201002</td>\n",
       "      <td>AK</td>\n",
-       "      <td>8912.0</td>\n",
-       "      <td>453411.0</td>\n",
+       "      <td>9040.0</td>\n",
+       "      <td>460250.0</td>\n",
       "      <td>NaN</td>\n",
-       "      <td>462323.0</td>\n",
-       "      <td>42.0</td>\n",
+       "      <td>469290.0</td>\n",
+       "      <td>41.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
-       "      <td>462323</td>\n",
-       "      <td>1</td>\n",
+       "      <td>469290</td>\n",
+       "      <td>0</td>\n",
       "      <td>0</td>\n",
-       "      <td>6fce5edde2805fba8ccfabc0232b39bb079a0c0a</td>\n",
+       "      <td>77ea689d232f21d7c3e86721a92703f37b818df6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
@@ -426,21 +426,21 @@
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
-       "      <td>20201001</td>\n",
+       "      <td>20201002</td>\n",
       "      <td>AL</td>\n",
-       "      <td>155744.0</td>\n",
-       "      <td>1000570.0</td>\n",
+       "      <td>156698.0</td>\n",
+       "      <td>1006612.0</td>\n",
       "      <td>NaN</td>\n",
-       "      <td>1138732.0</td>\n",
-       "      <td>760.0</td>\n",
-       "      <td>17257.0</td>\n",
+       "      <td>1145601.0</td>\n",
+       "      <td>752.0</td>\n",
+       "      <td>17420.0</td>\n",
       "      <td>NaN</td>\n",
-       "      <td>1832.0</td>\n",
+       "      <td>1840.0</td>\n",
       "      <td>...</td>\n",
-       "      <td>1156314</td>\n",
-       "      <td>8</td>\n",
-       "      <td>0</td>\n",
-       "      <td>d7277459136bf4c2075ceaea9cd64c6a193a9e80</td>\n",
+       "      <td>1163310</td>\n",
+       "      <td>2</td>\n",
+       "      <td>163</td>\n",
+       "      <td>38da52ec54bc8d9114fc1ef392985f137e8c9e1f</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
@@ -455,20 +455,20 @@
      ],
      "text/plain": [
       "       date state  positive   negative  pending  totalTestResults  \\\n",
-       "0  20201001    AK    8912.0   453411.0      NaN          462323.0   \n",
-       "1  20201001    AL  155744.0  1000570.0      NaN         1138732.0   \n",
+       "0  20201002    AK    9040.0   460250.0      NaN          469290.0   \n",
+       "1  20201002    AL  156698.0  1006612.0      NaN         1145601.0   \n",
       "\n",
       "   hospitalizedCurrently  hospitalizedCumulative  inIcuCurrently  \\\n",
-       "0                   42.0                     NaN             NaN   \n",
-       "1                  760.0                 17257.0             NaN   \n",
+       "0                   41.0                     NaN             NaN   \n",
+       "1                  752.0                 17420.0             NaN   \n",
       "\n",
       "   inIcuCumulative  ...   posNeg  deathIncrease  hospitalizedIncrease  \\\n",
-       "0              NaN  ...   462323              1                     0   \n",
-       "1           1832.0  ...  1156314              8                     0   \n",
+       "0              NaN  ...   469290              0                     0   \n",
+       "1           1840.0  ...  1163310              2                   163   \n",
       "\n",
       "                                       hash commercialScore  \\\n",
-       "0  6fce5edde2805fba8ccfabc0232b39bb079a0c0a               0   \n",
-       "1  d7277459136bf4c2075ceaea9cd64c6a193a9e80               0   \n",
+       "0  77ea689d232f21d7c3e86721a92703f37b818df6               0   \n",
+       "1  38da52ec54bc8d9114fc1ef392985f137e8c9e1f               0   \n",
       "\n",
       "  negativeRegularScore negativeScore  positiveScore  score grade  \n",
       "0                    0             0              0      0        \n",
@@ -508,18 +508,18 @@
   "version": "3.7.3"
  },
  "papermill": {
-   "duration": 34.808625,
-   "end_time": "2020-10-02T07:38:35.670354",
+   "duration": 36.095374,
+   "end_time": "2020-10-03T07:35:53.164442",
   "environment_variables": {},
   "exception": null,
-   "input_path": "/tmp/el6ip4bh/notebooks/process/download-covidtracking-data.ipynb",
+   "input_path": "/tmp/zy0hkj0c/notebooks/process/download-covidtracking-data.ipynb",
   "output_path": "runs/download-covidtracking-data.runs.ipynb",
   "parameters": {
-    "PAPERMILL_INPUT_PATH": "/tmp/el6ip4bh/notebooks/process/download-covidtracking-data.ipynb",
+    "PAPERMILL_INPUT_PATH": "/tmp/zy0hkj0c/notebooks/process/download-covidtracking-data.ipynb",
    "PAPERMILL_OUTPUT_PATH": "runs/download-covidtracking-data.runs.ipynb",
    "out_folder": "data/covidtracking"
   },
-   "start_time": "2020-10-02T07:38:00.861729",
+   "start_time": "2020-10-03T07:35:17.069068",
   "version": "1.1.0"
  }
 },

 %% Cell type:code id: tags:

 ``` python
 import requests
 import os
 import pandas as pd
 ```

 %% Cell type:code id: tags:parameters

 ``` python
 out_folder = "../data/covidtracking/"
 PAPERMILL_OUTPUT_PATH = None
 ```

 %% Cell type:code id: tags:injected-parameters

 ``` python
 # Parameters
-PAPERMILL_INPUT_PATH = "/tmp/el6ip4bh/notebooks/process/download-covidtracking-data.ipynb"
+PAPERMILL_INPUT_PATH = "/tmp/zy0hkj0c/notebooks/process/download-covidtracking-data.ipynb"
 PAPERMILL_OUTPUT_PATH = "runs/download-covidtracking-data.runs.ipynb"
 out_folder = "data/covidtracking"
 ```

 %% Cell type:markdown id: tags:

 # Download state metadata

 Download a dataset of URLs for data for each US state and several territories. See [Google Doc](https://docs.google.com/spreadsheets/d/18oVRrHj3c183mHmq3m89_163yuYltLNlOmPerQ18E8w/htmlview?sle=true).

 %% Cell type:code id: tags:

 ``` python
 url = 'http://covidtracking.com/api/states/info'
 r = requests.get(url, allow_redirects=True)
 states_metadata_json = r.content
 ```

 %% Cell type:code id: tags:

 ``` python
 # save the result
 if PAPERMILL_OUTPUT_PATH:
    out_path = os.path.join(out_folder, 'states-metadata.json')
    with open(out_path, 'wb') as f:
        f.write(states_metadata_json)
 ```

 %% Cell type:code id: tags:

 ``` python
 metadata_df = pd.read_json(states_metadata_json)
 print(len(metadata_df), "states and territories have metadata")
 metadata_df.head(2)
 ```

 %% Output

    56 states and territories have metadata

      state                                              notes  \
    0    AK  Alaska combines PCR and antigen tests in the t...
    1    AL  Alabama combines PCR and antigen tests in the ...
    
                                             covid19Site  \
    0  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...
    1  https://alpublichealth.maps.arcgis.com/apps/op...
    
                                    covid19SiteSecondary  \
    0  https://experience.arcgis.com/experience/ed1c8...
    1  https://alpublichealth.maps.arcgis.com/apps/op...
    
                                     covid19SiteTertiary covid19SiteQuaternary  \
    0  https://alaska-dhss.maps.arcgis.com/apps/opsda...
    1  https://services7.arcgis.com/4RQmZZ0yaZkGR1zy/...
    
      covid19SiteQuinary          twitter  \
    0                        @Alaska_DHSS
    1                     @alpublichealth
    
                                          covid19SiteOld  \
    0  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...
    1  http://www.alabamapublichealth.gov/infectiousd...
    
      covidTrackingProjectPreferredTotalTestUnits  \
    0                                   Specimens
    1                               Unclear units
    
      covidTrackingProjectPreferredTotalTestField totalTestResultsField     name  \
    0                             totalTestsViral     Total Tests (PCR)   Alaska
    1                             totalTestsViral     Total Tests (PCR)  Alabama
    
       fips pui    pum
    0     2      False
    1     1      False

 %% Cell type:markdown id: tags:

 # Download daily state data

 %% Cell type:code id: tags:

 ``` python
 url = 'https://covidtracking.com/api/states/daily'
 r = requests.get(url, allow_redirects=True)
 states_daily_json = r.content
 ```

 %% Cell type:code id: tags:

 ``` python
 # save the result
 if PAPERMILL_OUTPUT_PATH:
    out_path = os.path.join(out_folder, 'states-daily.json')
    with open(out_path, 'wb') as f:
        f.write(states_daily_json)
 ```

 %% Cell type:code id: tags:

 ``` python
 data_df = pd.read_json(states_daily_json)
 print(len(data_df), "data points")
 data_df.head(2)
 ```

 %% Output

-    11858 data points
+    11914 data points

           date state  positive   negative  pending  totalTestResults  \
-    0  20201001    AK    8912.0   453411.0      NaN          462323.0
-    1  20201001    AL  155744.0  1000570.0      NaN         1138732.0
+    0  20201002    AK    9040.0   460250.0      NaN          469290.0
+    1  20201002    AL  156698.0  1006612.0      NaN         1145601.0
    
       hospitalizedCurrently  hospitalizedCumulative  inIcuCurrently  \
-    0                   42.0                     NaN             NaN
-    1                  760.0                 17257.0             NaN
+    0                   41.0                     NaN             NaN
+    1                  752.0                 17420.0             NaN
    
       inIcuCumulative  ...   posNeg  deathIncrease  hospitalizedIncrease  \
-    0              NaN  ...   462323              1                     0
-    1           1832.0  ...  1156314              8                     0
+    0              NaN  ...   469290              0                     0
+    1           1840.0  ...  1163310              2                   163
    
                                           hash commercialScore  \
-    0  6fce5edde2805fba8ccfabc0232b39bb079a0c0a               0
-    1  d7277459136bf4c2075ceaea9cd64c6a193a9e80               0
+    0  77ea689d232f21d7c3e86721a92703f37b818df6               0
+    1  38da52ec54bc8d9114fc1ef392985f137e8c9e1f               0
    
      negativeRegularScore negativeScore  positiveScore  score grade
    0                    0             0              0      0
    1                    0             0              0      0
    
    [2 rows x 54 columns]