renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json

10d924ee · CI-bot · renku 0.10.4 · 8ab2ed32 · 10d924ee · 10d924ee
Commit 10d924ee authored 4 years ago by CI-bot Committed by renku 0.10.4 4 years ago
--- a/.renku/workflow/20c86d3d7885417b805f91afd932a627.cwl
+++ b/.renku/workflow/20c86d3d7885417b805f91afd932a627.cwl
+class: Workflow
+cwlVersion: v1.0
+hints: []
+inputs:
+  input_1:
+    default: out_folder
+    streamable: false
+    type: string
+  input_2:
+    default: data/covidtracking
+    streamable: false
+    type: string
+  input_3:
+    default:
+      class: File
+      path: ../../notebooks/process/download-covidtracking-data.ipynb
+    streamable: false
+    type: File
+  input_4:
+    default: runs/download-covidtracking-data.runs.ipynb
+    streamable: false
+    type: string
+  input_5:
+    default: states-daily.json
+    streamable: false
+    type: string
+  input_6:
+    default: states-metadata.json
+    streamable: false
+    type: string
+outputs:
+  output_0:
+    outputSource: step_1/output_1
+    streamable: false
+    type: Directory
+  output_1:
+    outputSource: step_1/output_0
+    streamable: false
+    type: File
+requirements: []
+steps:
+  step_1:
+    in:
+      input_1: input_1
+      input_2: input_2
+      input_3: input_3
+      input_4: input_4
+    out:
+    - output_0
+    - output_1
+    run: a17d560c41a54f5aa307ce5f3c5effe5_papermill.cwl
+  step_2:
+    in:
+      filename: input_5
+      input_directory: step_1/output_1
+    out:
+    - output_file
+    run:
+      arguments: []
+      baseCommand:
+      - 'true'
+      class: CommandLineTool
+      cwlVersion: v1.0
+      hints: []
+      inputs:
+        filename:
+          default: states-daily.json
+          streamable: false
+          type: string
+        input_directory:
+          streamable: false
+          type: Directory
+      outputs:
+        output_file:
+          outputBinding:
+            glob: $(inputs.filename)
+          streamable: false
+          type: File
+      permanentFailCodes: []
+      requirements:
+      - &id001
+        class: InlineJavascriptRequirement
+      - &id002
+        class: InitialWorkDirRequirement
+        listing: $(inputs.input_directory.listing)
+      successCodes: []
+      temporaryFailCodes: []
+  step_3:
+    in:
+      filename: input_6
+      input_directory: step_1/output_1
+    out:
+    - output_file
+    run:
+      arguments: []
+      baseCommand:
+      - 'true'
+      class: CommandLineTool
+      cwlVersion: v1.0
+      hints: []
+      inputs:
+        filename:
+          default: states-metadata.json
+          streamable: false
+          type: string
+        input_directory:
+          streamable: false
+          type: Directory
+      outputs:
+        output_file:
+          outputBinding:
+            glob: $(inputs.filename)
+          streamable: false
+          type: File
+      permanentFailCodes: []
+      requirements:
+      - *id001
+      - *id002
+      successCodes: []
+      temporaryFailCodes: []
--- a/data/covidtracking/states-daily.json
+++ b/data/covidtracking/states-daily.json
--- a/data/covidtracking/states-metadata.json
+++ b/data/covidtracking/states-metadata.json
--- a/runs/download-covidtracking-data.runs.ipynb
+++ b/runs/download-covidtracking-data.runs.ipynb
@@ -5,10 +5,10 @@
   "execution_count": 1,
   "metadata": {
    "papermill": {
-     "duration": 4.909303,
+     "duration": 2.138843,
-     "end_time": "2020-08-12T07:32:52.940667",
+     "end_time": "2020-08-13T07:32:08.076759",
     "exception": false,
-     "start_time": "2020-08-12T07:32:48.031364",
+     "start_time": "2020-08-13T07:32:05.937916",
     "status": "completed"
    },
    "tags": []
@@ -25,10 +25,10 @@
   "execution_count": 2,
   "metadata": {
    "papermill": {
-     "duration": 0.020786,
+     "duration": 0.019514,
-     "end_time": "2020-08-12T07:32:52.974199",
+     "end_time": "2020-08-13T07:32:08.109425",
     "exception": false,
-     "start_time": "2020-08-12T07:32:52.953413",
+     "start_time": "2020-08-13T07:32:08.089911",
     "status": "completed"
    },
    "tags": [
@@ -46,10 +46,10 @@
   "execution_count": 3,
   "metadata": {
    "papermill": {
-     "duration": 0.018808,
+     "duration": 0.020546,
-     "end_time": "2020-08-12T07:32:53.002848",
+     "end_time": "2020-08-13T07:32:08.138868",
     "exception": false,
-     "start_time": "2020-08-12T07:32:52.984040",
+     "start_time": "2020-08-13T07:32:08.118322",
     "status": "completed"
    },
    "tags": [
@@ -59,7 +59,7 @@
   "outputs": [],
   "source": [
    "# Parameters\n",
-    "PAPERMILL_INPUT_PATH = \"/tmp/fa34vlpr/notebooks/process/download-covidtracking-data.ipynb\"\n",
+    "PAPERMILL_INPUT_PATH = \"/tmp/8ktvuu43/notebooks/process/download-covidtracking-data.ipynb\"\n",
    "PAPERMILL_OUTPUT_PATH = \"runs/download-covidtracking-data.runs.ipynb\"\n",
    "out_folder = \"data/covidtracking\"\n"
   ]
@@ -68,10 +68,10 @@
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
-     "duration": 0.007114,
+     "duration": 0.007322,
-     "end_time": "2020-08-12T07:32:53.017421",
+     "end_time": "2020-08-13T07:32:08.155663",
     "exception": false,
-     "start_time": "2020-08-12T07:32:53.010307",
+     "start_time": "2020-08-13T07:32:08.148341",
     "status": "completed"
    },
    "tags": []
@@ -87,10 +87,10 @@
   "execution_count": 4,
   "metadata": {
    "papermill": {
-     "duration": 0.253919,
+     "duration": 0.288666,
-     "end_time": "2020-08-12T07:32:53.278771",
+     "end_time": "2020-08-13T07:32:08.451980",
     "exception": false,
-     "start_time": "2020-08-12T07:32:53.024852",
+     "start_time": "2020-08-13T07:32:08.163314",
     "status": "completed"
    },
    "tags": []
@@ -107,10 +107,10 @@
   "execution_count": 5,
   "metadata": {
    "papermill": {
-     "duration": 0.023702,
+     "duration": 0.021771,
-     "end_time": "2020-08-12T07:32:53.315338",
+     "end_time": "2020-08-13T07:32:08.485568",
     "exception": false,
-     "start_time": "2020-08-12T07:32:53.291636",
+     "start_time": "2020-08-13T07:32:08.463797",
     "status": "completed"
    },
    "tags": []
@@ -129,10 +129,10 @@
   "execution_count": 6,
   "metadata": {
    "papermill": {
-     "duration": 0.112023,
+     "duration": 0.075526,
-     "end_time": "2020-08-12T07:32:53.437535",
+     "end_time": "2020-08-13T07:32:08.570959",
     "exception": false,
-     "start_time": "2020-08-12T07:32:53.325512",
+     "start_time": "2020-08-13T07:32:08.495433",
     "status": "completed"
    },
    "tags": []
@@ -249,10 +249,10 @@
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
-     "duration": 0.010012,
+     "duration": 0.008029,
-     "end_time": "2020-08-12T07:32:53.458945",
+     "end_time": "2020-08-13T07:32:08.591772",
     "exception": false,
-     "start_time": "2020-08-12T07:32:53.448933",
+     "start_time": "2020-08-13T07:32:08.583743",
     "status": "completed"
    },
    "tags": []
@@ -266,10 +266,10 @@
   "execution_count": 7,
   "metadata": {
    "papermill": {
-     "duration": 0.847674,
+     "duration": 0.912442,
-     "end_time": "2020-08-12T07:32:54.316583",
+     "end_time": "2020-08-13T07:32:09.512304",
     "exception": false,
-     "start_time": "2020-08-12T07:32:53.468909",
+     "start_time": "2020-08-13T07:32:08.599862",
     "status": "completed"
    },
    "tags": []
@@ -286,10 +286,10 @@
   "execution_count": 8,
   "metadata": {
    "papermill": {
-     "duration": 0.034853,
+     "duration": 0.032626,
-     "end_time": "2020-08-12T07:32:54.363904",
+     "end_time": "2020-08-13T07:32:09.560190",
     "exception": false,
-     "start_time": "2020-08-12T07:32:54.329051",
+     "start_time": "2020-08-13T07:32:09.527564",
     "status": "completed"
    },
    "tags": []
@@ -308,10 +308,10 @@
   "execution_count": 9,
   "metadata": {
    "papermill": {
-     "duration": 0.359212,
+     "duration": 0.363787,
-     "end_time": "2020-08-12T07:32:54.737206",
+     "end_time": "2020-08-13T07:32:09.935148",
     "exception": false,
-     "start_time": "2020-08-12T07:32:54.377994",
+     "start_time": "2020-08-13T07:32:09.571361",
     "status": "completed"
    },
    "tags": []
@@ -321,7 +321,7 @@
     "name": "stdout",
     "output_type": "stream",
     "text": [
-      "8921 data points\n"
+      "8977 data points\n"
     ]
    },
    {
@@ -371,21 +371,21 @@
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
-       "      <td>20200811</td>\n",
+       "      <td>20200812</td>\n",
       "      <td>AK</td>\n",
-       "      <td>4589.0</td>\n",
+       "      <td>4655.0</td>\n",
-       "      <td>286155.0</td>\n",
+       "      <td>287927.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>39.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
-       "      <td>4.0</td>\n",
+       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
-       "      <td>290744</td>\n",
+       "      <td>292582</td>\n",
-       "      <td>0</td>\n",
+       "      <td>1</td>\n",
       "      <td>0</td>\n",
-       "      <td>75189f7e45df3d19bc61fc8790a6df68e33c41e4</td>\n",
+       "      <td>04d3f681722d466f15660e1ded53888693484520</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
@@ -395,21 +395,21 @@
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
-       "      <td>20200811</td>\n",
+       "      <td>20200812</td>\n",
       "      <td>AL</td>\n",
-       "      <td>103851.0</td>\n",
+       "      <td>104786.0</td>\n",
-       "      <td>679838.0</td>\n",
+       "      <td>690985.0</td>\n",
       "      <td>NaN</td>\n",
-       "      <td>1506.0</td>\n",
+       "      <td>1372.0</td>\n",
-       "      <td>12070.0</td>\n",
+       "      <td>12292.0</td>\n",
       "      <td>NaN</td>\n",
-       "      <td>1270.0</td>\n",
+       "      <td>1282.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
-       "      <td>783689</td>\n",
+       "      <td>795771</td>\n",
-       "      <td>50</td>\n",
+       "      <td>35</td>\n",
-       "      <td>0</td>\n",
+       "      <td>222</td>\n",
-       "      <td>d775daf03cf949c25340db800bb25f4eec3c5fda</td>\n",
+       "      <td>3b8e75d37adb9ba3951ec8b2aae85ba1fa9b02f1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
@@ -424,20 +424,20 @@
      ],
      "text/plain": [
       "       date state  positive  negative  pending  hospitalizedCurrently  \\\n",
-       "0  20200811    AK    4589.0  286155.0      NaN                   39.0   \n",
+       "0  20200812    AK    4655.0  287927.0      NaN                   39.0   \n",
-       "1  20200811    AL  103851.0  679838.0      NaN                 1506.0   \n",
+       "1  20200812    AL  104786.0  690985.0      NaN                 1372.0   \n",
       "\n",
       "   hospitalizedCumulative  inIcuCurrently  inIcuCumulative  \\\n",
       "0                     NaN             NaN              NaN   \n",
-       "1                 12070.0             NaN           1270.0   \n",
+       "1                 12292.0             NaN           1282.0   \n",
       "\n",
       "   onVentilatorCurrently  ...  posNeg  deathIncrease hospitalizedIncrease  \\\n",
-       "0                    4.0  ...  290744              0                    0   \n",
+       "0                    3.0  ...  292582              1                    0   \n",
-       "1                    NaN  ...  783689             50                    0   \n",
+       "1                    NaN  ...  795771             35                  222   \n",
       "\n",
       "                                       hash commercialScore  \\\n",
-       "0  75189f7e45df3d19bc61fc8790a6df68e33c41e4               0   \n",
+       "0  04d3f681722d466f15660e1ded53888693484520               0   \n",
-       "1  d775daf03cf949c25340db800bb25f4eec3c5fda               0   \n",
+       "1  3b8e75d37adb9ba3951ec8b2aae85ba1fa9b02f1               0   \n",
       "\n",
       "  negativeRegularScore  negativeScore  positiveScore score  grade  \n",
       "0                    0              0              0     0         \n",
@@ -477,18 +477,18 @@
   "version": "3.7.3"
  },
  "papermill": {
-   "duration": 8.638524,
+   "duration": 5.465167,
-   "end_time": "2020-08-12T07:32:55.057745",
+   "end_time": "2020-08-13T07:32:10.255042",
   "environment_variables": {},
   "exception": null,
-   "input_path": "/tmp/fa34vlpr/notebooks/process/download-covidtracking-data.ipynb",
+   "input_path": "/tmp/8ktvuu43/notebooks/process/download-covidtracking-data.ipynb",
   "output_path": "runs/download-covidtracking-data.runs.ipynb",
   "parameters": {
-    "PAPERMILL_INPUT_PATH": "/tmp/fa34vlpr/notebooks/process/download-covidtracking-data.ipynb",
+    "PAPERMILL_INPUT_PATH": "/tmp/8ktvuu43/notebooks/process/download-covidtracking-data.ipynb",
    "PAPERMILL_OUTPUT_PATH": "runs/download-covidtracking-data.runs.ipynb",
    "out_folder": "data/covidtracking"
   },
-   "start_time": "2020-08-12T07:32:46.419221",
+   "start_time": "2020-08-13T07:32:04.789875",
   "version": "1.1.0"
  }
 },

 %% Cell type:code id: tags:
 ``` python
 import requests
 import os
 import pandas as pd
 ```
 %% Cell type:code id: tags:parameters
 ``` python
 out_folder = "../data/covidtracking/"
 PAPERMILL_OUTPUT_PATH = None
 ```
 %% Cell type:code id: tags:injected-parameters
 ``` python
 # Parameters
-PAPERMILL_INPUT_PATH = "/tmp/fa34vlpr/notebooks/process/download-covidtracking-data.ipynb"
+PAPERMILL_INPUT_PATH = "/tmp/8ktvuu43/notebooks/process/download-covidtracking-data.ipynb"
 PAPERMILL_OUTPUT_PATH = "runs/download-covidtracking-data.runs.ipynb"
 out_folder = "data/covidtracking"
 ```
 %% Cell type:markdown id: tags:
 # Download state metadata
 Download a dataset of URLs for data for each US state and several territories. See [Google Doc](https://docs.google.com/spreadsheets/d/18oVRrHj3c183mHmq3m89_163yuYltLNlOmPerQ18E8w/htmlview?sle=true).
 %% Cell type:code id: tags:
 ``` python
 url = 'http://covidtracking.com/api/states/info'
 r = requests.get(url, allow_redirects=True)
 states_metadata_json = r.content
 ```
 %% Cell type:code id: tags:
 ``` python
 # save the result
 if PAPERMILL_OUTPUT_PATH:
    out_path = os.path.join(out_folder, 'states-metadata.json')
    with open(out_path, 'wb') as f:
        f.write(states_metadata_json)
 ```
 %% Cell type:code id: tags:
 ``` python
 metadata_df = pd.read_json(states_metadata_json)
 print(len(metadata_df), "states and territories have metadata")
 metadata_df.head(2)
 ```
 %% Output
    56 states and territories have metadata
      state                                              notes  \
    0    AK  Negatives = (Totals – Positives)\nPositives oc...
    1    AL  Negatives = (Totals - Positives) \nPositives o...
                                             covid19Site  \
    0  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...
    1  https://alpublichealth.maps.arcgis.com/apps/op...
                                    covid19SiteSecondary  \
    0  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...
    1            https://dph1.adph.state.al.us/covid-19/
                                     covid19SiteTertiary          twitter  \
    0  https://alaska-dhss.maps.arcgis.com/apps/opsda...     @Alaska_DHSS
    1                                                     @alpublichealth
                                          covid19SiteOld     name  fips pui    pum
    0  http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...   Alaska     2      False
    1  http://www.alabamapublichealth.gov/infectiousd...  Alabama     1      False
 %% Cell type:markdown id: tags:
 # Download daily state data
 %% Cell type:code id: tags:
 ``` python
 url = 'https://covidtracking.com/api/states/daily'
 r = requests.get(url, allow_redirects=True)
 states_daily_json = r.content
 ```
 %% Cell type:code id: tags:
 ``` python
 # save the result
 if PAPERMILL_OUTPUT_PATH:
    out_path = os.path.join(out_folder, 'states-daily.json')
    with open(out_path, 'wb') as f:
        f.write(states_daily_json)
 ```
 %% Cell type:code id: tags:
 ``` python
 data_df = pd.read_json(states_daily_json)
 print(len(data_df), "data points")
 data_df.head(2)
 ```
 %% Output
-    8921 data points
+    8977 data points
           date state  positive  negative  pending  hospitalizedCurrently  \
-    0  20200811    AK    4589.0  286155.0      NaN                   39.0
+    0  20200812    AK    4655.0  287927.0      NaN                   39.0
-    1  20200811    AL  103851.0  679838.0      NaN                 1506.0
+    1  20200812    AL  104786.0  690985.0      NaN                 1372.0
       hospitalizedCumulative  inIcuCurrently  inIcuCumulative  \
    0                     NaN             NaN              NaN
-    1                 12070.0             NaN           1270.0
+    1                 12292.0             NaN           1282.0
       onVentilatorCurrently  ...  posNeg  deathIncrease hospitalizedIncrease  \
-    0                    4.0  ...  290744              0                    0
+    0                    3.0  ...  292582              1                    0
-    1                    NaN  ...  783689             50                    0
+    1                    NaN  ...  795771             35                  222
                                           hash commercialScore  \
-    0  75189f7e45df3d19bc61fc8790a6df68e33c41e4               0
+    0  04d3f681722d466f15660e1ded53888693484520               0
-    1  d775daf03cf949c25340db800bb25f4eec3c5fda               0
+    1  3b8e75d37adb9ba3951ec8b2aae85ba1fa9b02f1               0
      negativeRegularScore  negativeScore  positiveScore score  grade
    0                    0              0              0     0
    1                    0              0              0     0
    [2 rows x 41 columns]