renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json

83095a6c · CI-bot · renku 0.10.4 · 3a3f5d65 · 83095a6c · 83095a6c
Commit 83095a6c authored 4 years ago by CI-bot Committed by renku 0.10.4 4 years ago
--- a/.renku/workflow/82240f0d1c284fb0a7acd54a9dcff0be.cwl
+++ b/.renku/workflow/82240f0d1c284fb0a7acd54a9dcff0be.cwl
+class: Workflow
+cwlVersion: v1.0
+hints: []
+inputs:
+  input_1:
+    default: out_folder
+    streamable: false
+    type: string
+  input_2:
+    default: data/covidtracking
+    streamable: false
+    type: string
+  input_3:
+    default:
+      class: File
+      path: ../../notebooks/process/download-covidtracking-data.ipynb
+    streamable: false
+    type: File
+  input_4:
+    default: runs/download-covidtracking-data.runs.ipynb
+    streamable: false
+    type: string
+  input_5:
+    default: states-metadata.json
+    streamable: false
+    type: string
+  input_6:
+    default: states-daily.json
+    streamable: false
+    type: string
+outputs:
+  output_0:
+    outputSource: step_1/output_1
+    streamable: false
+    type: Directory
+  output_2:
+    outputSource: step_1/output_0
+    streamable: false
+    type: File
+requirements: []
+steps:
+  step_1:
+    in:
+      input_1: input_1
+      input_2: input_2
+      input_3: input_3
+      input_4: input_4
+    out:
+    - output_0
+    - output_1
+    run: a17d560c41a54f5aa307ce5f3c5effe5_papermill.cwl
+  step_2:
+    in:
+      filename: input_5
+      input_directory: step_1/output_1
+    out:
+    - output_file
+    run:
+      arguments: []
+      baseCommand:
+      - 'true'
+      class: CommandLineTool
+      cwlVersion: v1.0
+      hints: []
+      inputs:
+        filename:
+          default: states-metadata.json
+          streamable: false
+          type: string
+        input_directory:
+          streamable: false
+          type: Directory
+      outputs:
+        output_file:
+          outputBinding:
+            glob: $(inputs.filename)
+          streamable: false
+          type: File
+      permanentFailCodes: []
+      requirements:
+      - &id001
+        class: InlineJavascriptRequirement
+      - &id002
+        class: InitialWorkDirRequirement
+        listing: $(inputs.input_directory.listing)
+      successCodes: []
+      temporaryFailCodes: []
+  step_3:
+    in:
+      filename: input_6
+      input_directory: step_1/output_1
+    out:
+    - output_file
+    run:
+      arguments: []
+      baseCommand:
+      - 'true'
+      class: CommandLineTool
+      cwlVersion: v1.0
+      hints: []
+      inputs:
+        filename:
+          default: states-daily.json
+          streamable: false
+          type: string
+        input_directory:
+          streamable: false
+          type: Directory
+      outputs:
+        output_file:
+          outputBinding:
+            glob: $(inputs.filename)
+          streamable: false
+          type: File
+      permanentFailCodes: []
+      requirements:
+      - *id001
+      - *id002
+      successCodes: []
+      temporaryFailCodes: []
--- a/data/covidtracking/states-daily.json
+++ b/data/covidtracking/states-daily.json
--- a/data/covidtracking/states-metadata.json
+++ b/data/covidtracking/states-metadata.json
--- a/runs/download-covidtracking-data.runs.ipynb
+++ b/runs/download-covidtracking-data.runs.ipynb
@@ -5,10 +5,10 @@
   "execution_count": 1,
   "metadata": {
    "papermill": {
-     "duration": 2.326882,
+     "duration": 4.902651,
-     "end_time": "2020-10-26T07:37:39.549797",
+     "end_time": "2020-10-26T10:16:32.908788",
     "exception": false,
-     "start_time": "2020-10-26T07:37:37.222915",
+     "start_time": "2020-10-26T10:16:28.006137",
     "status": "completed"
    },
    "tags": []
@@ -25,10 +25,10 @@
   "execution_count": 2,
   "metadata": {
    "papermill": {
-     "duration": 0.021897,
+     "duration": 0.021137,
-     "end_time": "2020-10-26T07:37:39.585932",
+     "end_time": "2020-10-26T10:16:32.941892",
     "exception": false,
-     "start_time": "2020-10-26T07:37:39.564035",
+     "start_time": "2020-10-26T10:16:32.920755",
     "status": "completed"
    },
    "tags": [
@@ -46,10 +46,10 @@
   "execution_count": 3,
   "metadata": {
    "papermill": {
-     "duration": 0.021802,
+     "duration": 0.022744,
-     "end_time": "2020-10-26T07:37:39.618175",
+     "end_time": "2020-10-26T10:16:32.974987",
     "exception": false,
-     "start_time": "2020-10-26T07:37:39.596373",
+     "start_time": "2020-10-26T10:16:32.952243",
     "status": "completed"
    },
    "tags": [
@@ -59,7 +59,7 @@
   "outputs": [],
   "source": [
    "# Parameters\n",
-    "PAPERMILL_INPUT_PATH = \"/tmp/nez13572/notebooks/process/download-covidtracking-data.ipynb\"\n",
+    "PAPERMILL_INPUT_PATH = \"/tmp/7ghx_g0f/notebooks/process/download-covidtracking-data.ipynb\"\n",
    "PAPERMILL_OUTPUT_PATH = \"runs/download-covidtracking-data.runs.ipynb\"\n",
    "out_folder = \"data/covidtracking\"\n"
   ]
@@ -68,10 +68,10 @@
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
-     "duration": 0.007886,
+     "duration": 0.007464,
-     "end_time": "2020-10-26T07:37:39.636816",
+     "end_time": "2020-10-26T10:16:32.992033",
     "exception": false,
-     "start_time": "2020-10-26T07:37:39.628930",
+     "start_time": "2020-10-26T10:16:32.984569",
     "status": "completed"
    },
    "tags": []
@@ -87,10 +87,10 @@
   "execution_count": 4,
   "metadata": {
    "papermill": {
-     "duration": 0.632897,
+     "duration": 0.195074,
-     "end_time": "2020-10-26T07:37:40.277250",
+     "end_time": "2020-10-26T10:16:33.194312",
     "exception": false,
-     "start_time": "2020-10-26T07:37:39.644353",
+     "start_time": "2020-10-26T10:16:32.999238",
     "status": "completed"
    },
    "tags": []
@@ -107,10 +107,10 @@
   "execution_count": 5,
   "metadata": {
    "papermill": {
-     "duration": 0.023332,
+     "duration": 0.024036,
-     "end_time": "2020-10-26T07:37:40.311285",
+     "end_time": "2020-10-26T10:16:33.230233",
     "exception": false,
-     "start_time": "2020-10-26T07:37:40.287953",
+     "start_time": "2020-10-26T10:16:33.206197",
     "status": "completed"
    },
    "tags": []
@@ -129,10 +129,10 @@
   "execution_count": 6,
   "metadata": {
    "papermill": {
-     "duration": 0.101663,
+     "duration": 0.074177,
-     "end_time": "2020-10-26T07:37:40.422853",
+     "end_time": "2020-10-26T10:16:33.314147",
     "exception": false,
-     "start_time": "2020-10-26T07:37:40.321190",
+     "start_time": "2020-10-26T10:16:33.239970",
     "status": "completed"
    },
    "tags": []
@@ -280,10 +280,10 @@
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
-     "duration": 0.00813,
+     "duration": 0.008562,
-     "end_time": "2020-10-26T07:37:40.440587",
+     "end_time": "2020-10-26T10:16:33.333915",
     "exception": false,
-     "start_time": "2020-10-26T07:37:40.432457",
+     "start_time": "2020-10-26T10:16:33.325353",
     "status": "completed"
    },
    "tags": []
@@ -297,10 +297,10 @@
   "execution_count": 7,
   "metadata": {
    "papermill": {
-     "duration": 29.828599,
+     "duration": 0.577776,
-     "end_time": "2020-10-26T07:38:10.278079",
+     "end_time": "2020-10-26T10:16:33.920363",
     "exception": false,
-     "start_time": "2020-10-26T07:37:40.449480",
+     "start_time": "2020-10-26T10:16:33.342587",
     "status": "completed"
    },
    "tags": []
@@ -317,10 +317,10 @@
   "execution_count": 8,
   "metadata": {
    "papermill": {
-     "duration": 0.060729,
+     "duration": 0.223258,
-     "end_time": "2020-10-26T07:38:10.353261",
+     "end_time": "2020-10-26T10:16:34.287657",
     "exception": false,
-     "start_time": "2020-10-26T07:38:10.292532",
+     "start_time": "2020-10-26T10:16:34.064399",
     "status": "completed"
    },
    "tags": []
@@ -339,10 +339,10 @@
   "execution_count": 9,
   "metadata": {
    "papermill": {
-     "duration": 0.666286,
+     "duration": 0.70868,
-     "end_time": "2020-10-26T07:38:11.030070",
+     "end_time": "2020-10-26T10:16:35.433160",
     "exception": false,
-     "start_time": "2020-10-26T07:38:10.363784",
+     "start_time": "2020-10-26T10:16:34.724480",
     "status": "completed"
    },
    "tags": []
@@ -508,18 +508,18 @@
   "version": "3.7.3"
  },
  "papermill": {
-   "duration": 35.451391,
+   "duration": 9.645951,
-   "end_time": "2020-10-26T07:38:11.351400",
+   "end_time": "2020-10-26T10:16:35.760040",
   "environment_variables": {},
   "exception": null,
-   "input_path": "/tmp/nez13572/notebooks/process/download-covidtracking-data.ipynb",
+   "input_path": "/tmp/7ghx_g0f/notebooks/process/download-covidtracking-data.ipynb",
   "output_path": "runs/download-covidtracking-data.runs.ipynb",
   "parameters": {
-    "PAPERMILL_INPUT_PATH": "/tmp/nez13572/notebooks/process/download-covidtracking-data.ipynb",
+    "PAPERMILL_INPUT_PATH": "/tmp/7ghx_g0f/notebooks/process/download-covidtracking-data.ipynb",
    "PAPERMILL_OUTPUT_PATH": "runs/download-covidtracking-data.runs.ipynb",
    "out_folder": "data/covidtracking"
   },
-   "start_time": "2020-10-26T07:37:35.900009",
+   "start_time": "2020-10-26T10:16:26.114089",
   "version": "1.1.0"
  }
 },

 %% Cell type:code id: tags:
 ``` python
 import requests
 import os
 import pandas as pd
 ```
 %% Cell type:code id: tags:parameters
 ``` python
 out_folder = "../data/covidtracking/"
 PAPERMILL_OUTPUT_PATH = None
 ```
 %% Cell type:code id: tags:injected-parameters
 ``` python
 # Parameters
-PAPERMILL_INPUT_PATH = "/tmp/nez13572/notebooks/process/download-covidtracking-data.ipynb"
+PAPERMILL_INPUT_PATH = "/tmp/7ghx_g0f/notebooks/process/download-covidtracking-data.ipynb"
 PAPERMILL_OUTPUT_PATH = "runs/download-covidtracking-data.runs.ipynb"
 out_folder = "data/covidtracking"
 ```
 %% Cell type:markdown id: tags:
 # Download state metadata
 Download a dataset of URLs for data for each US state and several territories. See [Google Doc](https://docs.google.com/spreadsheets/d/18oVRrHj3c183mHmq3m89_163yuYltLNlOmPerQ18E8w/htmlview?sle=true).
 %% Cell type:code id: tags:
 ``` python
 url = 'http://covidtracking.com/api/states/info'
 r = requests.get(url, allow_redirects=True)
 states_metadata_json = r.content
 ```
 %% Cell type:code id: tags:
 ``` python
 # save the result
 if PAPERMILL_OUTPUT_PATH:
    out_path = os.path.join(out_folder, 'states-metadata.json')
    with open(out_path, 'wb') as f:
        f.write(states_metadata_json)
 ```
 %% Cell type:code id: tags:
 ``` python
 metadata_df = pd.read_json(states_metadata_json)
 print(len(metadata_df), "states and territories have metadata")
 metadata_df.head(2)
 ```
 %% Output
    56 states and territories have metadata
      state                                              notes  \
    0    AK  Alaska combines PCR and antigen tests in the t...
    1    AL  Alabama combines PCR and antigen tests in the ...
                                             covid19Site  \
    0  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...
    1  https://alpublichealth.maps.arcgis.com/apps/op...
                                    covid19SiteSecondary  \
    0  https://experience.arcgis.com/experience/ed1c8...
    1  https://alpublichealth.maps.arcgis.com/apps/op...
                                     covid19SiteTertiary covid19SiteQuaternary  \
    0  https://alaska-dhss.maps.arcgis.com/apps/opsda...                  None
    1  https://services7.arcgis.com/4RQmZZ0yaZkGR1zy/...                  None
      covid19SiteQuinary          twitter  \
    0               None     @Alaska_DHSS
    1               None  @alpublichealth
                                          covid19SiteOld  \
    0  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...
    1  http://www.alabamapublichealth.gov/infectiousd...
      covidTrackingProjectPreferredTotalTestUnits  \
    0                                   Specimens
    1                               Unclear units
      covidTrackingProjectPreferredTotalTestField totalTestResultsField       pui  \
    0                             totalTestsViral     Total Tests (PCR)  All data
    1                             totalTestsViral     Total Tests (PCR)   No data
         pum     name  fips
    0  False   Alaska     2
    1  False  Alabama     1
 %% Cell type:markdown id: tags:
 # Download daily state data
 %% Cell type:code id: tags:
 ``` python
 url = 'https://covidtracking.com/api/states/daily'
 r = requests.get(url, allow_redirects=True)
 states_daily_json = r.content
 ```
 %% Cell type:code id: tags:
 ``` python
 # save the result
 if PAPERMILL_OUTPUT_PATH:
    out_path = os.path.join(out_folder, 'states-daily.json')
    with open(out_path, 'wb') as f:
        f.write(states_daily_json)
 ```
 %% Cell type:code id: tags:
 ``` python
 data_df = pd.read_json(states_daily_json)
 print(len(data_df), "data points")
 data_df.head(2)
 ```
 %% Output
    13213 data points
           date state  positive  probableCases   negative  pending  \
    0  20201025    AK   14064.0            NaN   545905.0      NaN
    1  20201025    AL  184355.0        26474.0  1144785.0      NaN
      totalTestResultsSource  totalTestResults  hospitalizedCurrently  \
    0        totalTestsViral          559969.0                   58.0
    1        totalTestsViral         1302666.0                  922.0
       hospitalizedCumulative  ...   posNeg  deathIncrease  hospitalizedIncrease  \
    0                     NaN  ...   559969              0                     0
    1                 19595.0  ...  1329140              0                     0
                                           hash  commercialScore  \
    0  175a4f70a15c098ac277246bff6ae33838f66029                0
    1  14421edcbba78df1217731644f92a3a03211c44d                0
      negativeRegularScore negativeScore positiveScore score  grade
    0                    0             0             0     0
    1                    0             0             0     0
    [2 rows x 55 columns]