diff --git a/.renku/workflow/04da715b913f47ce977e51c1ca5aaba7.cwl b/.renku/workflow/04da715b913f47ce977e51c1ca5aaba7.cwl
new file mode 100644
index 0000000000000000000000000000000000000000..4954d051a85321c882c22370c097db11e7654adc
--- /dev/null
+++ b/.renku/workflow/04da715b913f47ce977e51c1ca5aaba7.cwl
@@ -0,0 +1,120 @@
+class: Workflow
+cwlVersion: v1.0
+hints: []
+inputs:
+ input_1:
+ default: out_folder
+ streamable: false
+ type: string
+ input_2:
+ default: data/covidtracking
+ streamable: false
+ type: string
+ input_3:
+ default:
+ class: File
+ path: ../../notebooks/process/download-covidtracking-data.ipynb
+ streamable: false
+ type: File
+ input_4:
+ default: runs/download-covidtracking-data.runs.ipynb
+ streamable: false
+ type: string
+ input_5:
+ default: states-daily.json
+ streamable: false
+ type: string
+ input_6:
+ default: states-metadata.json
+ streamable: false
+ type: string
+outputs:
+ output_0:
+ outputSource: step_1/output_0
+ streamable: false
+ type: File
+ output_2:
+ outputSource: step_1/output_1
+ streamable: false
+ type: Directory
+requirements: []
+steps:
+ step_1:
+ in:
+ input_1: input_1
+ input_2: input_2
+ input_3: input_3
+ input_4: input_4
+ out:
+ - output_0
+ - output_1
+ run: a17d560c41a54f5aa307ce5f3c5effe5_papermill.cwl
+ step_2:
+ in:
+ filename: input_5
+ input_directory: step_1/output_1
+ out:
+ - output_file
+ run:
+ arguments: []
+ baseCommand:
+ - 'true'
+ class: CommandLineTool
+ cwlVersion: v1.0
+ hints: []
+ inputs:
+ filename:
+ default: states-daily.json
+ streamable: false
+ type: string
+ input_directory:
+ streamable: false
+ type: Directory
+ outputs:
+ output_file:
+ outputBinding:
+ glob: $(inputs.filename)
+ streamable: false
+ type: File
+ permanentFailCodes: []
+ requirements:
+ - &id001
+ class: InlineJavascriptRequirement
+ - &id002
+ class: InitialWorkDirRequirement
+ listing: $(inputs.input_directory.listing)
+ successCodes: []
+ temporaryFailCodes: []
+ step_3:
+ in:
+ filename: input_6
+ input_directory: step_1/output_1
+ out:
+ - output_file
+ run:
+ arguments: []
+ baseCommand:
+ - 'true'
+ class: CommandLineTool
+ cwlVersion: v1.0
+ hints: []
+ inputs:
+ filename:
+ default: states-metadata.json
+ streamable: false
+ type: string
+ input_directory:
+ streamable: false
+ type: Directory
+ outputs:
+ output_file:
+ outputBinding:
+ glob: $(inputs.filename)
+ streamable: false
+ type: File
+ permanentFailCodes: []
+ requirements:
+ - *id001
+ - *id002
+ successCodes: []
+ temporaryFailCodes: []
diff --git a/data/covidtracking/states-daily.json b/data/covidtracking/states-daily.json
index b256a0e80d31de419603f674dd24f5cc9e34b8e8..922002861cf4058844e9f10f0191966cb6401f0c 100644
--- a/data/covidtracking/states-daily.json
+++ b/data/covidtracking/states-daily.json
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:75361bc4be6af13f1c6d13fcf8e4782f87b35a8ddfc3a49e8d0b615dd2bb0f98
-size 15282956
+oid sha256:320eccb4de3249f45ec7cdc32eef1cf834d6ff25db8b5b7b2473f8544032be24
+size 15358340
diff --git a/data/covidtracking/states-metadata.json b/data/covidtracking/states-metadata.json
index 9437983d7573ef56f7baa20ea10e27416b7560be..439328a647eb144683ef3062b9195dde2e2add63 100644
--- a/data/covidtracking/states-metadata.json
+++ b/data/covidtracking/states-metadata.json
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:236847a50b3285ad90276f1b2cfa35f27f212b4bbd36a3d9b378a17a1836d013
-size 81459
+oid sha256:4a37e1c4dac9e1750d8113450091cec1decd4e34afb0bc0025e4e1af0429bc9f
+size 86082
diff --git a/runs/download-covidtracking-data.runs.ipynb b/runs/download-covidtracking-data.runs.ipynb
index 28add9f860efa8aa842c88b5e5f40d2503671383..eea624b32cf6c4e3fd2983fc2c44ea79d0ac3ad8 100644
--- a/runs/download-covidtracking-data.runs.ipynb
+++ b/runs/download-covidtracking-data.runs.ipynb
@@ -5,10 +5,10 @@
"execution_count": 1,
"metadata": {
"papermill": {
- "duration": 1.612677,
- "end_time": "2020-09-25T07:34:42.022198",
+ "duration": 2.162702,
+ "end_time": "2020-09-26T07:35:07.425341",
"exception": false,
- "start_time": "2020-09-25T07:34:40.409521",
+ "start_time": "2020-09-26T07:35:05.262639",
"status": "completed"
},
"tags": []
@@ -25,10 +25,10 @@
"execution_count": 2,
"metadata": {
"papermill": {
- "duration": 0.020045,
- "end_time": "2020-09-25T07:34:42.053328",
+ "duration": 0.021606,
+ "end_time": "2020-09-26T07:35:07.457180",
"exception": false,
- "start_time": "2020-09-25T07:34:42.033283",
+ "start_time": "2020-09-26T07:35:07.435574",
"status": "completed"
},
"tags": [
@@ -46,10 +46,10 @@
"execution_count": 3,
"metadata": {
"papermill": {
- "duration": 0.019397,
- "end_time": "2020-09-25T07:34:42.080862",
+ "duration": 0.021266,
+ "end_time": "2020-09-26T07:35:07.487485",
"exception": false,
- "start_time": "2020-09-25T07:34:42.061465",
+ "start_time": "2020-09-26T07:35:07.466219",
"status": "completed"
},
"tags": [
@@ -59,7 +59,7 @@
"outputs": [],
"source": [
"# Parameters\n",
- "PAPERMILL_INPUT_PATH = \"/tmp/mnc0qhdm/notebooks/process/download-covidtracking-data.ipynb\"\n",
+ "PAPERMILL_INPUT_PATH = \"/tmp/0kos9h6c/notebooks/process/download-covidtracking-data.ipynb\"\n",
"PAPERMILL_OUTPUT_PATH = \"runs/download-covidtracking-data.runs.ipynb\"\n",
"out_folder = \"data/covidtracking\"\n"
]
@@ -68,10 +68,10 @@
"cell_type": "markdown",
"metadata": {
"papermill": {
- "duration": 0.008287,
- "end_time": "2020-09-25T07:34:42.097619",
+ "duration": 0.008912,
+ "end_time": "2020-09-26T07:35:07.508256",
"exception": false,
- "start_time": "2020-09-25T07:34:42.089332",
+ "start_time": "2020-09-26T07:35:07.499344",
"status": "completed"
},
"tags": []
@@ -87,10 +87,10 @@
"execution_count": 4,
"metadata": {
"papermill": {
- "duration": 0.648039,
- "end_time": "2020-09-25T07:34:42.752897",
+ "duration": 0.599096,
+ "end_time": "2020-09-26T07:35:08.114521",
"exception": false,
- "start_time": "2020-09-25T07:34:42.104858",
+ "start_time": "2020-09-26T07:35:07.515425",
"status": "completed"
},
"tags": []
@@ -107,10 +107,10 @@
"execution_count": 5,
"metadata": {
"papermill": {
- "duration": 0.023293,
- "end_time": "2020-09-25T07:34:42.787240",
+ "duration": 0.025711,
+ "end_time": "2020-09-26T07:35:08.154668",
"exception": false,
- "start_time": "2020-09-25T07:34:42.763947",
+ "start_time": "2020-09-26T07:35:08.128957",
"status": "completed"
},
"tags": []
@@ -129,10 +129,10 @@
"execution_count": 6,
"metadata": {
"papermill": {
- "duration": 0.079676,
- "end_time": "2020-09-25T07:34:42.878894",
+ "duration": 0.067298,
+ "end_time": "2020-09-26T07:35:08.232175",
"exception": false,
- "start_time": "2020-09-25T07:34:42.799218",
+ "start_time": "2020-09-26T07:35:08.164877",
"status": "completed"
},
"tags": []
@@ -171,6 +171,8 @@
" <th>covid19Site</th>\n",
" <th>covid19SiteSecondary</th>\n",
" <th>covid19SiteTertiary</th>\n",
+ " <th>covid19SiteQuaternary</th>\n",
+ " <th>covid19SiteQuinary</th>\n",
" <th>twitter</th>\n",
" <th>covid19SiteOld</th>\n",
" <th>covidTrackingProjectPreferredTotalTestUnits</th>\n",
@@ -190,6 +192,8 @@
" <td>http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...</td>\n",
" <td>https://experience.arcgis.com/experience/ed1c8...</td>\n",
" <td>https://alaska-dhss.maps.arcgis.com/apps/opsda...</td>\n",
+ " <td></td>\n",
+ " <td></td>\n",
" <td>@Alaska_DHSS</td>\n",
" <td>http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-...</td>\n",
" <td>Specimens</td>\n",
@@ -207,6 +211,8 @@
" <td>https://alpublichealth.maps.arcgis.com/apps/op...</td>\n",
" <td>https://alpublichealth.maps.arcgis.com/apps/op...</td>\n",
" <td>https://services7.arcgis.com/4RQmZZ0yaZkGR1zy/...</td>\n",
+ " <td></td>\n",
+ " <td></td>\n",
" <td>@alpublichealth</td>\n",
" <td>http://www.alabamapublichealth.gov/infectiousd...</td>\n",
" <td>Unclear units</td>\n",
@@ -234,9 +240,13 @@
"0 https://experience.arcgis.com/experience/ed1c8... \n",
"1 https://alpublichealth.maps.arcgis.com/apps/op... \n",
"\n",
- " covid19SiteTertiary twitter \\\n",
- "0 https://alaska-dhss.maps.arcgis.com/apps/opsda... @Alaska_DHSS \n",
- "1 https://services7.arcgis.com/4RQmZZ0yaZkGR1zy/... @alpublichealth \n",
+ " covid19SiteTertiary covid19SiteQuaternary \\\n",
+ "0 https://alaska-dhss.maps.arcgis.com/apps/opsda... \n",
+ "1 https://services7.arcgis.com/4RQmZZ0yaZkGR1zy/... \n",
+ "\n",
+ " covid19SiteQuinary twitter \\\n",
+ "0 @Alaska_DHSS \n",
+ "1 @alpublichealth \n",
"\n",
" covid19SiteOld \\\n",
"0 http://dhss.alaska.gov/dph/Epi/id/Pages/COVID-... \n",
@@ -270,10 +280,10 @@
"cell_type": "markdown",
"metadata": {
"papermill": {
- "duration": 0.007996,
- "end_time": "2020-09-25T07:34:42.898203",
+ "duration": 0.008867,
+ "end_time": "2020-09-26T07:35:08.254764",
"exception": false,
- "start_time": "2020-09-25T07:34:42.890207",
+ "start_time": "2020-09-26T07:35:08.245897",
"status": "completed"
},
"tags": []
@@ -287,10 +297,10 @@
"execution_count": 7,
"metadata": {
"papermill": {
- "duration": 2.599604,
- "end_time": "2020-09-25T07:34:45.506376",
+ "duration": 25.621929,
+ "end_time": "2020-09-26T07:35:33.885260",
"exception": false,
- "start_time": "2020-09-25T07:34:42.906772",
+ "start_time": "2020-09-26T07:35:08.263331",
"status": "completed"
},
"tags": []
@@ -307,10 +317,10 @@
"execution_count": 8,
"metadata": {
"papermill": {
- "duration": 0.041541,
- "end_time": "2020-09-25T07:34:45.562155",
+ "duration": 0.039913,
+ "end_time": "2020-09-26T07:35:33.936689",
"exception": false,
- "start_time": "2020-09-25T07:34:45.520614",
+ "start_time": "2020-09-26T07:35:33.896776",
"status": "completed"
},
"tags": []
@@ -329,10 +339,10 @@
"execution_count": 9,
"metadata": {
"papermill": {
- "duration": 0.676157,
- "end_time": "2020-09-25T07:34:46.249595",
+ "duration": 0.555361,
+ "end_time": "2020-09-26T07:35:34.504488",
"exception": false,
- "start_time": "2020-09-25T07:34:45.573438",
+ "start_time": "2020-09-26T07:35:33.949127",
"status": "completed"
},
"tags": []
@@ -342,7 +352,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "11466 data points\n"
+ "11522 data points\n"
]
},
{
@@ -392,21 +402,21 @@
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
- " <td>20200924</td>\n",
+ " <td>20200925</td>\n",
" <td>AK</td>\n",
- " <td>7941.0</td>\n",
- " <td>425257.0</td>\n",
+ " <td>8202.0</td>\n",
+ " <td>433130.0</td>\n",
" <td>NaN</td>\n",
- " <td>433198.0</td>\n",
+ " <td>441332.0</td>\n",
" <td>43.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>...</td>\n",
- " <td>433198</td>\n",
- " <td>0</td>\n",
+ " <td>441332</td>\n",
+ " <td>6</td>\n",
" <td>0</td>\n",
- " <td>b83363bdfb4b0d19318d00489c44e906270412f1</td>\n",
+ " <td>0271fcc794329b2dd035402cb74aad56124c446b</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
@@ -416,21 +426,21 @@
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
- " <td>20200924</td>\n",
+ " <td>20200925</td>\n",
" <td>AL</td>\n",
- " <td>148206.0</td>\n",
- " <td>944747.0</td>\n",
+ " <td>150658.0</td>\n",
+ " <td>963364.0</td>\n",
" <td>NaN</td>\n",
- " <td>1078180.0</td>\n",
- " <td>744.0</td>\n",
- " <td>16778.0</td>\n",
+ " <td>1097595.0</td>\n",
+ " <td>718.0</td>\n",
+ " <td>16852.0</td>\n",
" <td>NaN</td>\n",
- " <td>1679.0</td>\n",
+ " <td>1782.0</td>\n",
" <td>...</td>\n",
- " <td>1092953</td>\n",
- " <td>18</td>\n",
- " <td>80</td>\n",
- " <td>4b1b271eafc1e79991c11cae326c455bd34ae063</td>\n",
+ " <td>1114022</td>\n",
+ " <td>-15</td>\n",
+ " <td>74</td>\n",
+ " <td>08184faa52c95520b2871a34cbea8adad1c1d054</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
@@ -445,20 +455,20 @@
],
"text/plain": [
" date state positive negative pending totalTestResults \\\n",
- "0 20200924 AK 7941.0 425257.0 NaN 433198.0 \n",
- "1 20200924 AL 148206.0 944747.0 NaN 1078180.0 \n",
+ "0 20200925 AK 8202.0 433130.0 NaN 441332.0 \n",
+ "1 20200925 AL 150658.0 963364.0 NaN 1097595.0 \n",
"\n",
" hospitalizedCurrently hospitalizedCumulative inIcuCurrently \\\n",
"0 43.0 NaN NaN \n",
- "1 744.0 16778.0 NaN \n",
+ "1 718.0 16852.0 NaN \n",
"\n",
" inIcuCumulative ... posNeg deathIncrease hospitalizedIncrease \\\n",
- "0 NaN ... 433198 0 0 \n",
- "1 1679.0 ... 1092953 18 80 \n",
+ "0 NaN ... 441332 6 0 \n",
+ "1 1782.0 ... 1114022 -15 74 \n",
"\n",
" hash commercialScore \\\n",
- "0 b83363bdfb4b0d19318d00489c44e906270412f1 0 \n",
- "1 4b1b271eafc1e79991c11cae326c455bd34ae063 0 \n",
+ "0 0271fcc794329b2dd035402cb74aad56124c446b 0 \n",
+ "1 08184faa52c95520b2871a34cbea8adad1c1d054 0 \n",
"\n",
" negativeRegularScore negativeScore positiveScore score grade \n",
"0 0 0 0 0 \n",
@@ -498,18 +508,18 @@
"version": "3.7.3"
},
"papermill": {
- "duration": 7.556729,
- "end_time": "2020-09-25T07:34:46.571455",
+ "duration": 30.804134,
+ "end_time": "2020-09-26T07:35:34.828167",
"environment_variables": {},
"exception": null,
- "input_path": "/tmp/mnc0qhdm/notebooks/process/download-covidtracking-data.ipynb",
+ "input_path": "/tmp/0kos9h6c/notebooks/process/download-covidtracking-data.ipynb",
"output_path": "runs/download-covidtracking-data.runs.ipynb",
"parameters": {
- "PAPERMILL_INPUT_PATH": "/tmp/mnc0qhdm/notebooks/process/download-covidtracking-data.ipynb",
+ "PAPERMILL_INPUT_PATH": "/tmp/0kos9h6c/notebooks/process/download-covidtracking-data.ipynb",
"PAPERMILL_OUTPUT_PATH": "runs/download-covidtracking-data.runs.ipynb",
"out_folder": "data/covidtracking"
},
- "start_time": "2020-09-25T07:34:39.014726",
+ "start_time": "2020-09-26T07:35:04.024033",
"version": "1.1.0"
}
},