renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json

c2470f17 · CI-bot · renku 0.10.4 · 1c2ccfb6 · c2470f17 · c2470f17
Commit c2470f17 authored 4 years ago by CI-bot Committed by renku 0.10.4 4 years ago
--- a/.renku/workflow/483e40a495904a2288be9d726c9b92c5.cwl
+++ b/.renku/workflow/483e40a495904a2288be9d726c9b92c5.cwl
+class: Workflow
+cwlVersion: v1.0
+hints: []
+inputs:
+  input_1:
+    default: states-daily.json
+    streamable: false
+    type: string
+  input_2:
+    default: states-metadata.json
+    streamable: false
+    type: string
+  input_3:
+    default: out_folder
+    streamable: false
+    type: string
+  input_4:
+    default: data/covidtracking
+    streamable: false
+    type: string
+  input_5:
+    default:
+      class: File
+      path: ../../notebooks/process/download-covidtracking-data.ipynb
+    streamable: false
+    type: File
+  input_6:
+    default: runs/download-covidtracking-data.runs.ipynb
+    streamable: false
+    type: string
+outputs:
+  output_1:
+    outputSource: step_3/output_0
+    streamable: false
+    type: File
+  output_2:
+    outputSource: step_3/output_1
+    streamable: false
+    type: Directory
+requirements: []
+steps:
+  step_1:
+    in:
+      filename: input_1
+      input_directory: step_3/output_1
+    out:
+    - output_file
+    run:
+      arguments: []
+      baseCommand:
+      - 'true'
+      class: CommandLineTool
+      cwlVersion: v1.0
+      hints: []
+      inputs:
+        filename:
+          default: states-daily.json
+          streamable: false
+          type: string
+        input_directory:
+          streamable: false
+          type: Directory
+      outputs:
+        output_file:
+          outputBinding:
+            glob: $(inputs.filename)
+          streamable: false
+          type: File
+      permanentFailCodes: []
+      requirements:
+      - &id001
+        class: InlineJavascriptRequirement
+      - &id002
+        class: InitialWorkDirRequirement
+        listing: $(inputs.input_directory.listing)
+      successCodes: []
+      temporaryFailCodes: []
+  step_2:
+    in:
+      filename: input_2
+      input_directory: step_3/output_1
+    out:
+    - output_file
+    run:
+      arguments: []
+      baseCommand:
+      - 'true'
+      class: CommandLineTool
+      cwlVersion: v1.0
+      hints: []
+      inputs:
+        filename:
+          default: states-metadata.json
+          streamable: false
+          type: string
+        input_directory:
+          streamable: false
+          type: Directory
+      outputs:
+        output_file:
+          outputBinding:
+            glob: $(inputs.filename)
+          streamable: false
+          type: File
+      permanentFailCodes: []
+      requirements:
+      - *id001
+      - *id002
+      successCodes: []
+      temporaryFailCodes: []
+  step_3:
+    in:
+      input_1: input_3
+      input_2: input_4
+      input_3: input_5
+      input_4: input_6
+    out:
+    - output_1
+    - output_0
+    run: a17d560c41a54f5aa307ce5f3c5effe5_papermill.cwl
--- a/data/covidtracking/states-daily.json
+++ b/data/covidtracking/states-daily.json
--- a/data/covidtracking/states-metadata.json
+++ b/data/covidtracking/states-metadata.json
--- a/runs/download-covidtracking-data.runs.ipynb
+++ b/runs/download-covidtracking-data.runs.ipynb
@@ -5,10 +5,10 @@
   "execution_count": 1,
   "metadata": {
    "papermill": {
-     "duration": 4.756697,
-     "end_time": "2020-12-01T07:37:01.782160",
+     "duration": 3.93015,
+     "end_time": "2020-12-02T07:37:54.293969",
     "exception": false,
-     "start_time": "2020-12-01T07:36:57.025463",
+     "start_time": "2020-12-02T07:37:50.363819",
     "status": "completed"
    },
    "tags": []
@@ -25,10 +25,10 @@
   "execution_count": 2,
   "metadata": {
    "papermill": {
-     "duration": 0.020999,
-     "end_time": "2020-12-01T07:37:01.816589",
+     "duration": 0.021148,
+     "end_time": "2020-12-02T07:37:54.329512",
     "exception": false,
-     "start_time": "2020-12-01T07:37:01.795590",
+     "start_time": "2020-12-02T07:37:54.308364",
     "status": "completed"
    },
    "tags": [
@@ -46,10 +46,10 @@
   "execution_count": 3,
   "metadata": {
    "papermill": {
-     "duration": 0.019808,
-     "end_time": "2020-12-01T07:37:01.844709",
+     "duration": 0.01668,
+     "end_time": "2020-12-02T07:37:54.353751",
     "exception": false,
-     "start_time": "2020-12-01T07:37:01.824901",
+     "start_time": "2020-12-02T07:37:54.337071",
     "status": "completed"
    },
    "tags": [
@@ -59,7 +59,7 @@
   "outputs": [],
   "source": [
    "# Parameters\n",
-    "PAPERMILL_INPUT_PATH = \"/tmp/c1zx9gql/notebooks/process/download-covidtracking-data.ipynb\"\n",
+    "PAPERMILL_INPUT_PATH = \"/tmp/0vxtf18s/notebooks/process/download-covidtracking-data.ipynb\"\n",
    "PAPERMILL_OUTPUT_PATH = \"runs/download-covidtracking-data.runs.ipynb\"\n",
    "out_folder = \"data/covidtracking\"\n"
   ]
@@ -68,10 +68,10 @@
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
-     "duration": 0.00769,
-     "end_time": "2020-12-01T07:37:01.860374",
+     "duration": 0.005936,
+     "end_time": "2020-12-02T07:37:54.365726",
     "exception": false,
-     "start_time": "2020-12-01T07:37:01.852684",
+     "start_time": "2020-12-02T07:37:54.359790",
     "status": "completed"
    },
    "tags": []
@@ -87,10 +87,10 @@
   "execution_count": 4,
   "metadata": {
    "papermill": {
-     "duration": 0.335211,
-     "end_time": "2020-12-01T07:37:02.201150",
+     "duration": 0.458783,
+     "end_time": "2020-12-02T07:37:54.829423",
     "exception": false,
-     "start_time": "2020-12-01T07:37:01.865939",
+     "start_time": "2020-12-02T07:37:54.370640",
     "status": "completed"
    },
    "tags": []
@@ -107,10 +107,10 @@
   "execution_count": 5,
   "metadata": {
    "papermill": {
-     "duration": 0.025666,
-     "end_time": "2020-12-01T07:37:02.240440",
+     "duration": 0.022083,
+     "end_time": "2020-12-02T07:37:54.864086",
     "exception": false,
-     "start_time": "2020-12-01T07:37:02.214774",
+     "start_time": "2020-12-02T07:37:54.842003",
     "status": "completed"
    },
    "tags": []
@@ -129,10 +129,10 @@
   "execution_count": 6,
   "metadata": {
    "papermill": {
-     "duration": 0.069108,
-     "end_time": "2020-12-01T07:37:02.314783",
+     "duration": 0.066974,
+     "end_time": "2020-12-02T07:37:54.940047",
     "exception": false,
-     "start_time": "2020-12-01T07:37:02.245675",
+     "start_time": "2020-12-02T07:37:54.873073",
     "status": "completed"
    },
    "tags": []
@@ -284,10 +284,10 @@
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
-     "duration": 0.007566,
-     "end_time": "2020-12-01T07:37:02.334631",
+     "duration": 0.007203,
+     "end_time": "2020-12-02T07:37:54.959979",
     "exception": false,
-     "start_time": "2020-12-01T07:37:02.327065",
+     "start_time": "2020-12-02T07:37:54.952776",
     "status": "completed"
    },
    "tags": []
@@ -301,10 +301,10 @@
   "execution_count": 7,
   "metadata": {
    "papermill": {
-     "duration": 0.672308,
-     "end_time": "2020-12-01T07:37:03.012521",
+     "duration": 0.62676,
+     "end_time": "2020-12-02T07:37:55.592981",
     "exception": false,
-     "start_time": "2020-12-01T07:37:02.340213",
+     "start_time": "2020-12-02T07:37:54.966221",
     "status": "completed"
    },
    "tags": []
@@ -321,10 +321,10 @@
   "execution_count": 8,
   "metadata": {
    "papermill": {
-     "duration": 0.048797,
-     "end_time": "2020-12-01T07:37:03.078266",
+     "duration": 0.043914,
+     "end_time": "2020-12-02T07:37:55.652028",
     "exception": false,
-     "start_time": "2020-12-01T07:37:03.029469",
+     "start_time": "2020-12-02T07:37:55.608114",
     "status": "completed"
    },
    "tags": []
@@ -343,10 +343,10 @@
   "execution_count": 9,
   "metadata": {
    "papermill": {
-     "duration": 0.560952,
-     "end_time": "2020-12-01T07:37:03.651492",
+     "duration": 0.517154,
+     "end_time": "2020-12-02T07:37:56.182763",
     "exception": false,
-     "start_time": "2020-12-01T07:37:03.090540",
+     "start_time": "2020-12-02T07:37:55.665609",
     "status": "completed"
    },
    "tags": []
@@ -356,7 +356,7 @@
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "15297 data points\n"
+      "15353 data points\n"
     ]
    },
    {
@@ -406,21 +406,21 @@
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
-       "      <td>20201130</td>\n",
+       "      <td>20201201</td>\n",
       "      <td>AK</td>\n",
-       "      <td>31323.0</td>\n",
+       "      <td>31834.0</td>\n",
       "      <td>NaN</td>\n",
-       "      <td>980073.0</td>\n",
+       "      <td>986794.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>totalTestsViral</td>\n",
-       "      <td>1011396.0</td>\n",
-       "      <td>162.0</td>\n",
-       "      <td>725.0</td>\n",
+       "      <td>1018628.0</td>\n",
+       "      <td>155.0</td>\n",
+       "      <td>749.0</td>\n",
       "      <td>...</td>\n",
-       "      <td>1011396</td>\n",
-       "      <td>0</td>\n",
-       "      <td>3</td>\n",
-       "      <td>271d3c383c856d19d56e16fbd4efca0f728d6f5a</td>\n",
+       "      <td>1018628</td>\n",
+       "      <td>1</td>\n",
+       "      <td>24</td>\n",
+       "      <td>6c0d9dbaedf0d16c7d345f9928e37ce2dba3a0ec</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
@@ -430,21 +430,21 @@
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
-       "      <td>20201130</td>\n",
+       "      <td>20201201</td>\n",
       "      <td>AL</td>\n",
-       "      <td>249524.0</td>\n",
-       "      <td>41501.0</td>\n",
-       "      <td>1376324.0</td>\n",
+       "      <td>252900.0</td>\n",
+       "      <td>42863.0</td>\n",
+       "      <td>1383805.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>totalTestsViral</td>\n",
-       "      <td>1584347.0</td>\n",
-       "      <td>1717.0</td>\n",
-       "      <td>25338.0</td>\n",
+       "      <td>1593842.0</td>\n",
+       "      <td>1785.0</td>\n",
+       "      <td>25610.0</td>\n",
       "      <td>...</td>\n",
-       "      <td>1625848</td>\n",
-       "      <td>1</td>\n",
-       "      <td>668</td>\n",
-       "      <td>1d4953d87fd314f6eb3ff436b379ce3b6ff4fab7</td>\n",
+       "      <td>1636705</td>\n",
+       "      <td>60</td>\n",
+       "      <td>272</td>\n",
+       "      <td>ff146d1c2df88ca7b0ac8e94dacc310424c5f0ba</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
@@ -459,20 +459,20 @@
      ],
      "text/plain": [
       "       date state  positive  probableCases   negative  pending  \\\n",
-       "0  20201130    AK   31323.0            NaN   980073.0      NaN   \n",
-       "1  20201130    AL  249524.0        41501.0  1376324.0      NaN   \n",
+       "0  20201201    AK   31834.0            NaN   986794.0      NaN   \n",
+       "1  20201201    AL  252900.0        42863.0  1383805.0      NaN   \n",
       "\n",
       "  totalTestResultsSource  totalTestResults  hospitalizedCurrently  \\\n",
-       "0        totalTestsViral         1011396.0                  162.0   \n",
-       "1        totalTestsViral         1584347.0                 1717.0   \n",
+       "0        totalTestsViral         1018628.0                  155.0   \n",
+       "1        totalTestsViral         1593842.0                 1785.0   \n",
       "\n",
       "   hospitalizedCumulative  ...   posNeg  deathIncrease  hospitalizedIncrease  \\\n",
-       "0                   725.0  ...  1011396              0                     3   \n",
-       "1                 25338.0  ...  1625848              1                   668   \n",
+       "0                   749.0  ...  1018628              1                    24   \n",
+       "1                 25610.0  ...  1636705             60                   272   \n",
       "\n",
       "                                       hash  commercialScore  \\\n",
-       "0  271d3c383c856d19d56e16fbd4efca0f728d6f5a                0   \n",
-       "1  1d4953d87fd314f6eb3ff436b379ce3b6ff4fab7                0   \n",
+       "0  6c0d9dbaedf0d16c7d345f9928e37ce2dba3a0ec                0   \n",
+       "1  ff146d1c2df88ca7b0ac8e94dacc310424c5f0ba                0   \n",
       "\n",
       "  negativeRegularScore negativeScore positiveScore score  grade  \n",
       "0                    0             0             0     0         \n",
@@ -512,18 +512,18 @@
   "version": "3.7.3"
  },
  "papermill": {
-   "duration": 8.757211,
-   "end_time": "2020-12-01T07:37:03.973005",
+   "duration": 8.024748,
+   "end_time": "2020-12-02T07:37:56.505279",
   "environment_variables": {},
   "exception": null,
-   "input_path": "/tmp/c1zx9gql/notebooks/process/download-covidtracking-data.ipynb",
+   "input_path": "/tmp/0vxtf18s/notebooks/process/download-covidtracking-data.ipynb",
   "output_path": "runs/download-covidtracking-data.runs.ipynb",
   "parameters": {
-    "PAPERMILL_INPUT_PATH": "/tmp/c1zx9gql/notebooks/process/download-covidtracking-data.ipynb",
+    "PAPERMILL_INPUT_PATH": "/tmp/0vxtf18s/notebooks/process/download-covidtracking-data.ipynb",
    "PAPERMILL_OUTPUT_PATH": "runs/download-covidtracking-data.runs.ipynb",
    "out_folder": "data/covidtracking"
   },
-   "start_time": "2020-12-01T07:36:55.215794",
+   "start_time": "2020-12-02T07:37:48.480531",
   "version": "1.1.0"
  }
 },

 %% Cell type:code id: tags:

 ``` python
 import requests
 import os
 import pandas as pd
 ```

 %% Cell type:code id: tags:parameters

 ``` python
 out_folder = "../data/covidtracking/"
 PAPERMILL_OUTPUT_PATH = None
 ```

 %% Cell type:code id: tags:injected-parameters

 ``` python
 # Parameters
-PAPERMILL_INPUT_PATH = "/tmp/c1zx9gql/notebooks/process/download-covidtracking-data.ipynb"
+PAPERMILL_INPUT_PATH = "/tmp/0vxtf18s/notebooks/process/download-covidtracking-data.ipynb"
 PAPERMILL_OUTPUT_PATH = "runs/download-covidtracking-data.runs.ipynb"
 out_folder = "data/covidtracking"
 ```

 %% Cell type:markdown id: tags:

 # Download state metadata

 Download a dataset of URLs for data for each US state and several territories. See [Google Doc](https://docs.google.com/spreadsheets/d/18oVRrHj3c183mHmq3m89_163yuYltLNlOmPerQ18E8w/htmlview?sle=true).

 %% Cell type:code id: tags:

 ``` python
 url = 'http://covidtracking.com/api/states/info'
 r = requests.get(url, allow_redirects=True)
 states_metadata_json = r.content
 ```

 %% Cell type:code id: tags:

 ``` python
 # save the result
 if PAPERMILL_OUTPUT_PATH:
    out_path = os.path.join(out_folder, 'states-metadata.json')
    with open(out_path, 'wb') as f:
        f.write(states_metadata_json)
 ```

 %% Cell type:code id: tags:

 ``` python
 metadata_df = pd.read_json(states_metadata_json)
 print(len(metadata_df), "states and territories have metadata")
 metadata_df.head(2)
 ```

 %% Output

    56 states and territories have metadata

      state                                              notes  \
    0    AK  Alaska combines PCR and antigen tests in the t...
    1    AL  Alabama combines PCR and antigen tests in the ...
    
                                             covid19Site  \
    0  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...
    1  https://alpublichealth.maps.arcgis.com/apps/op...
    
                                    covid19SiteSecondary  \
    0  https://experience.arcgis.com/experience/ed1c8...
    1  https://alpublichealth.maps.arcgis.com/apps/op...
    
                                     covid19SiteTertiary  \
    0  https://alaska-dhss.maps.arcgis.com/apps/opsda...
    1  https://services7.arcgis.com/4RQmZZ0yaZkGR1zy/...
    
                                   covid19SiteQuaternary  \
    0  https://services1.arcgis.com/WzFsmainVTuD5KML/...
    1                                               None
    
                                      covid19SiteQuinary          twitter  \
    0  https://services1.arcgis.com/WzFsmainVTuD5KML/...     @Alaska_DHSS
    1                                               None  @alpublichealth
    
                                          covid19SiteOld  \
    0  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...
    1  http://www.alabamapublichealth.gov/infectiousd...
    
      covidTrackingProjectPreferredTotalTestUnits  \
    0                                   Specimens
    1                               Unclear units
    
      covidTrackingProjectPreferredTotalTestField totalTestResultsField       pui  \
    0                             totalTestsViral     Total Tests (PCR)  All data
    1                             totalTestsViral     Total Tests (PCR)   No data
    
         pum     name  fips
    0  False   Alaska     2
    1  False  Alabama     1

 %% Cell type:markdown id: tags:

 # Download daily state data

 %% Cell type:code id: tags:

 ``` python
 url = 'https://covidtracking.com/api/states/daily'
 r = requests.get(url, allow_redirects=True)
 states_daily_json = r.content
 ```

 %% Cell type:code id: tags:

 ``` python
 # save the result
 if PAPERMILL_OUTPUT_PATH:
    out_path = os.path.join(out_folder, 'states-daily.json')
    with open(out_path, 'wb') as f:
        f.write(states_daily_json)
 ```

 %% Cell type:code id: tags:

 ``` python
 data_df = pd.read_json(states_daily_json)
 print(len(data_df), "data points")
 data_df.head(2)
 ```

 %% Output

-    15297 data points
+    15353 data points

           date state  positive  probableCases   negative  pending  \
-    0  20201130    AK   31323.0            NaN   980073.0      NaN
-    1  20201130    AL  249524.0        41501.0  1376324.0      NaN
+    0  20201201    AK   31834.0            NaN   986794.0      NaN
+    1  20201201    AL  252900.0        42863.0  1383805.0      NaN
    
      totalTestResultsSource  totalTestResults  hospitalizedCurrently  \
-    0        totalTestsViral         1011396.0                  162.0
-    1        totalTestsViral         1584347.0                 1717.0
+    0        totalTestsViral         1018628.0                  155.0
+    1        totalTestsViral         1593842.0                 1785.0
    
       hospitalizedCumulative  ...   posNeg  deathIncrease  hospitalizedIncrease  \
-    0                   725.0  ...  1011396              0                     3
-    1                 25338.0  ...  1625848              1                   668
+    0                   749.0  ...  1018628              1                    24
+    1                 25610.0  ...  1636705             60                   272
    
                                           hash  commercialScore  \
-    0  271d3c383c856d19d56e16fbd4efca0f728d6f5a                0
-    1  1d4953d87fd314f6eb3ff436b379ce3b6ff4fab7                0
+    0  6c0d9dbaedf0d16c7d345f9928e37ce2dba3a0ec                0
+    1  ff146d1c2df88ca7b0ac8e94dacc310424c5f0ba                0
    
      negativeRegularScore negativeScore positiveScore score  grade
    0                    0             0             0     0
    1                    0             0             0     0
    
    [2 rows x 55 columns]