From b907252d22dcbdb4acbafc7417f4105606a92769 Mon Sep 17 00:00:00 2001
From: "CR (covid cron)" <cramakri+covid-cron@ethz.ch>
Date: Fri, 20 Mar 2020 21:45:31 +0000
Subject: [PATCH] renku rerun data/covidtracking/states-metadata.json
data/covidtracking/states-daily.json
---
.../199a2fe6993a45bfba1bab2b6bb02cbc.cwl | 120 ++++++++++++++++
data/covidtracking/states-daily.json | 4 +-
data/covidtracking/states-metadata.json | 4 +-
runs/download-covidtracking-data.runs.ipynb | 130 +++++++++---------
4 files changed, 189 insertions(+), 69 deletions(-)
create mode 100644 .renku/workflow/199a2fe6993a45bfba1bab2b6bb02cbc.cwl
diff --git a/.renku/workflow/199a2fe6993a45bfba1bab2b6bb02cbc.cwl b/.renku/workflow/199a2fe6993a45bfba1bab2b6bb02cbc.cwl
new file mode 100644
index 00000000..d3b122aa
--- /dev/null
+++ b/.renku/workflow/199a2fe6993a45bfba1bab2b6bb02cbc.cwl
@@ -0,0 +1,120 @@
+class: Workflow
+cwlVersion: v1.0
+hints: []
+inputs:
+ input_1:
+ default: out_folder
+ streamable: false
+ type: string
+ input_2:
+ default: data/covidtracking
+ streamable: false
+ type: string
+ input_3:
+ default:
+ class: File
+ path: ../../notebooks/process/download-covidtracking-data.ipynb
+ streamable: false
+ type: File
+ input_4:
+ default: runs/download-covidtracking-data.runs.ipynb
+ streamable: false
+ type: string
+ input_5:
+ default: states-daily.json
+ streamable: false
+ type: string
+ input_6:
+ default: states-metadata.json
+ streamable: false
+ type: string
+outputs:
+ output_1:
+ outputSource: step_1/output_0
+ streamable: false
+ type: File
+ output_2:
+ outputSource: step_1/output_1
+ streamable: false
+ type: Directory
+requirements: []
+steps:
+ step_1:
+ in:
+ input_1: input_1
+ input_2: input_2
+ input_3: input_3
+ input_4: input_4
+ out:
+ - output_0
+ - output_1
+ run: ff5f474d38144c5d882c0036bd1059c2_papermill.cwl
+ step_2:
+ in:
+ filename: input_5
+ input_directory: step_1/output_1
+ out:
+ - output_file
+ run:
+ arguments: []
+ baseCommand:
+ - 'true'
+ class: CommandLineTool
+ cwlVersion: v1.0
+ hints: []
+ inputs:
+ filename:
+ default: states-daily.json
+ streamable: false
+ type: string
+ input_directory:
+ streamable: false
+ type: Directory
+ outputs:
+ output_file:
+ outputBinding:
+ glob: $(inputs.filename)
+ streamable: false
+ type: File
+ permanentFailCodes: []
+ requirements:
+ - &id001
+ class: InlineJavascriptRequirement
+ - &id002
+ class: InitialWorkDirRequirement
+ listing: $(inputs.input_directory.listing)
+ successCodes: []
+ temporaryFailCodes: []
+ step_3:
+ in:
+ filename: input_6
+ input_directory: step_1/output_1
+ out:
+ - output_file
+ run:
+ arguments: []
+ baseCommand:
+ - 'true'
+ class: CommandLineTool
+ cwlVersion: v1.0
+ hints: []
+ inputs:
+ filename:
+ default: states-metadata.json
+ streamable: false
+ type: string
+ input_directory:
+ streamable: false
+ type: Directory
+ outputs:
+ output_file:
+ outputBinding:
+ glob: $(inputs.filename)
+ streamable: false
+ type: File
+ permanentFailCodes: []
+ requirements:
+ - *id001
+ - *id002
+ successCodes: []
+ temporaryFailCodes: []
diff --git a/data/covidtracking/states-daily.json b/data/covidtracking/states-daily.json
index d54ef635..f2d6a22d 100644
--- a/data/covidtracking/states-daily.json
+++ b/data/covidtracking/states-daily.json
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:487f9dcaed9e5a43fa3f6685a7b7e8f6c3fb98b5658debf8ef23a758a84f2962
-size 101680
+oid sha256:9dd4d36f37d438f6cd6f77857cbd7cfbcc50bd154dcfe661b82f7b3cfbf04292
+size 109274
diff --git a/data/covidtracking/states-metadata.json b/data/covidtracking/states-metadata.json
index d2c8a6c4..1f1aca43 100644
--- a/data/covidtracking/states-metadata.json
+++ b/data/covidtracking/states-metadata.json
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:e6bb5b35059c3d346ca35ae90e2512bfacff2621fb4d747cda217799d2832b38
-size 21884
+oid sha256:313f39a8ca639fbe07354929d7fa1350f81131a25f5650d4c16ce733a3349b9f
+size 24242
diff --git a/runs/download-covidtracking-data.runs.ipynb b/runs/download-covidtracking-data.runs.ipynb
index 086bdde7..e80d21dc 100644
--- a/runs/download-covidtracking-data.runs.ipynb
+++ b/runs/download-covidtracking-data.runs.ipynb
@@ -5,10 +5,10 @@
"execution_count": 1,
"metadata": {
"papermill": {
- "duration": 0.582742,
- "end_time": "2020-03-20T09:12:16.167813",
+ "duration": 3.757919,
+ "end_time": "2020-03-20T21:45:29.726180",
"exception": false,
- "start_time": "2020-03-20T09:12:15.585071",
+ "start_time": "2020-03-20T21:45:25.968261",
"status": "completed"
},
"tags": []
@@ -25,10 +25,10 @@
"execution_count": 2,
"metadata": {
"papermill": {
- "duration": 0.017681,
- "end_time": "2020-03-20T09:12:16.194097",
+ "duration": 0.018354,
+ "end_time": "2020-03-20T21:45:29.758115",
"exception": false,
- "start_time": "2020-03-20T09:12:16.176416",
+ "start_time": "2020-03-20T21:45:29.739761",
"status": "completed"
},
"tags": [
@@ -46,10 +46,10 @@
"execution_count": 3,
"metadata": {
"papermill": {
- "duration": 0.019941,
- "end_time": "2020-03-20T09:12:16.222810",
+ "duration": 0.021368,
+ "end_time": "2020-03-20T21:45:29.787594",
"exception": false,
- "start_time": "2020-03-20T09:12:16.202869",
+ "start_time": "2020-03-20T21:45:29.766226",
"status": "completed"
},
"tags": [
@@ -59,7 +59,7 @@
"outputs": [],
"source": [
"# Parameters\n",
- "PAPERMILL_INPUT_PATH = \"/tmp/tq93huw7/notebooks/process/download-covidtracking-data.ipynb\"\n",
+ "PAPERMILL_INPUT_PATH = \"/tmp/f3pmvx85/notebooks/process/download-covidtracking-data.ipynb\"\n",
"PAPERMILL_OUTPUT_PATH = \"runs/download-covidtracking-data.runs.ipynb\"\n",
"out_folder = \"data/covidtracking\"\n"
]
@@ -68,10 +68,10 @@
"cell_type": "markdown",
"metadata": {
"papermill": {
- "duration": 0.006852,
- "end_time": "2020-03-20T09:12:16.239661",
+ "duration": 0.008377,
+ "end_time": "2020-03-20T21:45:29.807144",
"exception": false,
- "start_time": "2020-03-20T09:12:16.232809",
+ "start_time": "2020-03-20T21:45:29.798767",
"status": "completed"
},
"tags": []
@@ -87,10 +87,10 @@
"execution_count": 4,
"metadata": {
"papermill": {
- "duration": 1.543652,
- "end_time": "2020-03-20T09:12:17.791818",
+ "duration": 0.430485,
+ "end_time": "2020-03-20T21:45:30.245726",
"exception": false,
- "start_time": "2020-03-20T09:12:16.248166",
+ "start_time": "2020-03-20T21:45:29.815241",
"status": "completed"
},
"tags": []
@@ -107,10 +107,10 @@
"execution_count": 5,
"metadata": {
"papermill": {
- "duration": 0.019479,
- "end_time": "2020-03-20T09:12:17.820781",
+ "duration": 0.021358,
+ "end_time": "2020-03-20T21:45:30.277302",
"exception": false,
- "start_time": "2020-03-20T09:12:17.801302",
+ "start_time": "2020-03-20T21:45:30.255944",
"status": "completed"
},
"tags": []
@@ -129,10 +129,10 @@
"execution_count": 6,
"metadata": {
"papermill": {
- "duration": 0.067162,
- "end_time": "2020-03-20T09:12:17.897197",
+ "duration": 0.063279,
+ "end_time": "2020-03-20T21:45:30.348415",
"exception": false,
- "start_time": "2020-03-20T09:12:17.830035",
+ "start_time": "2020-03-20T21:45:30.285136",
"status": "completed"
},
"tags": []
@@ -183,7 +183,7 @@
" <td>AK</td>\n",
" <td>http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...</td>\n",
" <td>http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...</td>\n",
- " <td>NaN</td>\n",
+ " <td>http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...</td>\n",
" <td>@Alaska_DHSS</td>\n",
" <td>All data</td>\n",
" <td>False</td>\n",
@@ -195,11 +195,11 @@
" <td>AL</td>\n",
" <td>http://www.alabamapublichealth.gov/infectiousd...</td>\n",
" <td>https://alpublichealth.maps.arcgis.com/apps/op...</td>\n",
- " <td>NaN</td>\n",
+ " <td>None</td>\n",
" <td>@alpublichealth</td>\n",
" <td>No data</td>\n",
" <td>False</td>\n",
- " <td>Last negative count from 3/16.</td>\n",
+ " <td>Last negative count from 3/16. Last update tim...</td>\n",
" <td>Alabama</td>\n",
" </tr>\n",
" </tbody>\n",
@@ -211,17 +211,17 @@
"0 AK http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-... \n",
"1 AL http://www.alabamapublichealth.gov/infectiousd... \n",
"\n",
- " covid19Site covid19SiteSecondary \\\n",
- "0 http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-... NaN \n",
- "1 https://alpublichealth.maps.arcgis.com/apps/op... NaN \n",
+ " covid19Site \\\n",
+ "0 http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-... \n",
+ "1 https://alpublichealth.maps.arcgis.com/apps/op... \n",
"\n",
- " twitter pui pum \\\n",
- "0 @Alaska_DHSS All data False \n",
- "1 @alpublichealth No data False \n",
+ " covid19SiteSecondary twitter \\\n",
+ "0 http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-... @Alaska_DHSS \n",
+ "1 None @alpublichealth \n",
"\n",
- " notes name \n",
- "0 Unclear if their reported number means \"person... Alaska \n",
- "1 Last negative count from 3/16. Alabama "
+ " pui pum notes name \n",
+ "0 All data False Unclear if their reported number means \"person... Alaska \n",
+ "1 No data False Last negative count from 3/16. Last update tim... Alabama "
]
},
"execution_count": 6,
@@ -239,10 +239,10 @@
"cell_type": "markdown",
"metadata": {
"papermill": {
- "duration": 0.010887,
- "end_time": "2020-03-20T09:12:17.919775",
+ "duration": 0.008215,
+ "end_time": "2020-03-20T21:45:30.366192",
"exception": false,
- "start_time": "2020-03-20T09:12:17.908888",
+ "start_time": "2020-03-20T21:45:30.357977",
"status": "completed"
},
"tags": []
@@ -256,10 +256,10 @@
"execution_count": 7,
"metadata": {
"papermill": {
- "duration": 0.134228,
- "end_time": "2020-03-20T09:12:18.065352",
+ "duration": 0.658874,
+ "end_time": "2020-03-20T21:45:31.033153",
"exception": false,
- "start_time": "2020-03-20T09:12:17.931124",
+ "start_time": "2020-03-20T21:45:30.374279",
"status": "completed"
},
"tags": []
@@ -276,10 +276,10 @@
"execution_count": 8,
"metadata": {
"papermill": {
- "duration": 0.02218,
- "end_time": "2020-03-20T09:12:18.097912",
+ "duration": 0.018873,
+ "end_time": "2020-03-20T21:45:31.062632",
"exception": false,
- "start_time": "2020-03-20T09:12:18.075732",
+ "start_time": "2020-03-20T21:45:31.043759",
"status": "completed"
},
"tags": []
@@ -298,10 +298,10 @@
"execution_count": 9,
"metadata": {
"papermill": {
- "duration": 0.058368,
- "end_time": "2020-03-20T09:12:18.166166",
+ "duration": 0.053619,
+ "end_time": "2020-03-20T21:45:31.127152",
"exception": false,
- "start_time": "2020-03-20T09:12:18.107798",
+ "start_time": "2020-03-20T21:45:31.073533",
"status": "completed"
},
"tags": []
@@ -311,7 +311,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "757 data points\n"
+ "813 data points\n"
]
},
{
@@ -348,25 +348,25 @@
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
- " <td>20200319</td>\n",
+ " <td>20200320</td>\n",
" <td>AK</td>\n",
- " <td>6</td>\n",
- " <td>400.0</td>\n",
+ " <td>12</td>\n",
+ " <td>686.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
- " <td>406</td>\n",
- " <td>2020-03-19T20:00:00Z</td>\n",
+ " <td>698</td>\n",
+ " <td>2020-03-20T20:00:00Z</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
- " <td>20200319</td>\n",
+ " <td>20200320</td>\n",
" <td>AL</td>\n",
- " <td>68</td>\n",
+ " <td>81</td>\n",
" <td>28.0</td>\n",
" <td>NaN</td>\n",
" <td>0.0</td>\n",
- " <td>96</td>\n",
- " <td>2020-03-19T20:00:00Z</td>\n",
+ " <td>109</td>\n",
+ " <td>2020-03-20T20:00:00Z</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
@@ -374,12 +374,12 @@
],
"text/plain": [
" date state positive negative pending death total \\\n",
- "0 20200319 AK 6 400.0 NaN NaN 406 \n",
- "1 20200319 AL 68 28.0 NaN 0.0 96 \n",
+ "0 20200320 AK 12 686.0 NaN NaN 698 \n",
+ "1 20200320 AL 81 28.0 NaN 0.0 109 \n",
"\n",
" dateChecked \n",
- "0 2020-03-19T20:00:00Z \n",
- "1 2020-03-19T20:00:00Z "
+ "0 2020-03-20T20:00:00Z \n",
+ "1 2020-03-20T20:00:00Z "
]
},
"execution_count": 9,
@@ -413,18 +413,18 @@
"version": "3.7.3"
},
"papermill": {
- "duration": 3.92686,
- "end_time": "2020-03-20T09:12:18.486365",
+ "duration": 6.856589,
+ "end_time": "2020-03-20T21:45:31.451326",
"environment_variables": {},
"exception": null,
- "input_path": "/tmp/tq93huw7/notebooks/process/download-covidtracking-data.ipynb",
+ "input_path": "/tmp/f3pmvx85/notebooks/process/download-covidtracking-data.ipynb",
"output_path": "runs/download-covidtracking-data.runs.ipynb",
"parameters": {
- "PAPERMILL_INPUT_PATH": "/tmp/tq93huw7/notebooks/process/download-covidtracking-data.ipynb",
+ "PAPERMILL_INPUT_PATH": "/tmp/f3pmvx85/notebooks/process/download-covidtracking-data.ipynb",
"PAPERMILL_OUTPUT_PATH": "runs/download-covidtracking-data.runs.ipynb",
"out_folder": "data/covidtracking"
},
- "start_time": "2020-03-20T09:12:14.559505",
+ "start_time": "2020-03-20T21:45:24.594737",
"version": "1.1.0"
}
},
--
GitLab