From b6f4a4fe7443f40f580e8ee5201a7c6f21890e11 Mon Sep 17 00:00:00 2001
From: Chandrasekhar Ramakrishnan <cramakri@ethz.ch>
Date: Mon, 23 Mar 2020 22:44:18 +0000
Subject: [PATCH] renku run papermill --report-mode -p ts_folder
./data/covid-19_jhu-csse/ -p rates_folder ./data/covid-19_rates/ -p
geodata_path ./data/geodata/geo_data.csv --inject-paths
notebooks/Dashboard.ipynb runs/Dashboard.run.ipynb
---
...00d9c26c4cc58c1123c8280b555d_papermill.cwl | 117 ++++++++++
runs/Dashboard.run.ipynb | 220 +++++++++++-------
2 files changed, 253 insertions(+), 84 deletions(-)
create mode 100644 .renku/workflow/283500d9c26c4cc58c1123c8280b555d_papermill.cwl
diff --git a/.renku/workflow/283500d9c26c4cc58c1123c8280b555d_papermill.cwl b/.renku/workflow/283500d9c26c4cc58c1123c8280b555d_papermill.cwl
new file mode 100644
index 00000000..77af68be
--- /dev/null
+++ b/.renku/workflow/283500d9c26c4cc58c1123c8280b555d_papermill.cwl
@@ -0,0 +1,117 @@
+arguments:
+- position: 1
+ separate: true
+ shellQuote: true
+ valueFrom: --report-mode
+baseCommand:
+- papermill
+class: CommandLineTool
+cwlVersion: v1.0
+hints: []
+inputs:
+ input_2:
+ default: ts_folder
+ inputBinding:
+ position: 2
+ prefix: -p
+ separate: true
+ shellQuote: true
+ streamable: false
+ type: string
+ input_3:
+ default:
+ class: Directory
+ listing: []
+ path: ../../data/covid-19_jhu-csse
+ inputBinding:
+ position: 3
+ separate: true
+ shellQuote: true
+ streamable: false
+ type: Directory
+ input_4:
+ default: rates_folder
+ inputBinding:
+ position: 4
+ prefix: -p
+ separate: true
+ shellQuote: true
+ streamable: false
+ type: string
+ input_5:
+ default:
+ class: Directory
+ listing: []
+ path: ../../data/covid-19_rates
+ inputBinding:
+ position: 5
+ separate: true
+ shellQuote: true
+ streamable: false
+ type: Directory
+ input_6:
+ default: geodata_path
+ inputBinding:
+ position: 6
+ prefix: -p
+ separate: true
+ shellQuote: true
+ streamable: false
+ type: string
+ input_7:
+ default:
+ class: File
+ path: ../../data/geodata/geo_data.csv
+ inputBinding:
+ position: 7
+ separate: true
+ shellQuote: true
+ streamable: false
+ type: File
+ input_8:
+ default:
+ class: File
+ path: ../../notebooks/Dashboard.ipynb
+ inputBinding:
+ position: 8
+ prefix: --inject-paths
+ separate: true
+ shellQuote: true
+ streamable: false
+ type: File
+ input_9:
+ default: runs/Dashboard.run.ipynb
+ inputBinding:
+ position: 9
+ separate: true
+ shellQuote: true
+ streamable: false
+ type: string
+outputs:
+ output_0:
+ outputBinding:
+ glob: $(inputs.input_9)
+ streamable: false
+ type: File
+permanentFailCodes: []
+requirements:
+- class: InlineJavascriptRequirement
+- class: InitialWorkDirRequirement
+ listing:
+ - entry: '$({"listing": [], "class": "Directory"})'
+ entryname: runs
+ writable: true
+ - entry: $(inputs.input_3)
+ entryname: data/covid-19_jhu-csse
+ writable: false
+ - entry: $(inputs.input_5)
+ entryname: data/covid-19_rates
+ writable: false
+ - entry: $(inputs.input_7)
+ entryname: data/geodata/geo_data.csv
+ writable: false
+ - entry: $(inputs.input_8)
+ entryname: notebooks/Dashboard.ipynb
+ writable: false
+successCodes: []
+temporaryFailCodes: []
diff --git a/runs/Dashboard.run.ipynb b/runs/Dashboard.run.ipynb
index acfb54b9..5891796b 100644
--- a/runs/Dashboard.run.ipynb
+++ b/runs/Dashboard.run.ipynb
@@ -4,11 +4,14 @@
"cell_type": "code",
"execution_count": 1,
"metadata": {
+ "jupyter": {
+ "source_hidden": true
+ },
"papermill": {
- "duration": 0.809594,
- "end_time": "2020-03-23T08:23:17.020112",
+ "duration": 0.502795,
+ "end_time": "2020-03-23T22:44:15.528530",
"exception": false,
- "start_time": "2020-03-23T08:23:16.210518",
+ "start_time": "2020-03-23T22:44:15.025735",
"status": "completed"
},
"tags": []
@@ -26,11 +29,14 @@
"cell_type": "code",
"execution_count": 2,
"metadata": {
+ "jupyter": {
+ "source_hidden": true
+ },
"papermill": {
- "duration": 0.0267,
- "end_time": "2020-03-23T08:23:17.067998",
+ "duration": 0.018207,
+ "end_time": "2020-03-23T22:44:15.557722",
"exception": false,
- "start_time": "2020-03-23T08:23:17.041298",
+ "start_time": "2020-03-23T22:44:15.539515",
"status": "completed"
},
"tags": [
@@ -50,11 +56,14 @@
"cell_type": "code",
"execution_count": 3,
"metadata": {
+ "jupyter": {
+ "source_hidden": true
+ },
"papermill": {
- "duration": 0.030646,
- "end_time": "2020-03-23T08:23:17.112804",
+ "duration": 0.019523,
+ "end_time": "2020-03-23T22:44:15.588368",
"exception": false,
- "start_time": "2020-03-23T08:23:17.082158",
+ "start_time": "2020-03-23T22:44:15.568845",
"status": "completed"
},
"tags": [
@@ -64,22 +73,25 @@
"outputs": [],
"source": [
"# Parameters\n",
- "PAPERMILL_INPUT_PATH = \"/tmp/zlogjitb/notebooks/Dashboard.ipynb\"\n",
+ "PAPERMILL_INPUT_PATH = \"notebooks/Dashboard.ipynb\"\n",
"PAPERMILL_OUTPUT_PATH = \"runs/Dashboard.run.ipynb\"\n",
- "ts_folder = \"/tmp/zlogjitb/data/covid-19_jhu-csse\"\n",
- "rates_folder = \"/tmp/zlogjitb/data/covid-19_rates\"\n",
- "geodata_path = \"/tmp/zlogjitb/data/geodata/geo_data.csv\"\n"
+ "ts_folder = \"./data/covid-19_jhu-csse/\"\n",
+ "rates_folder = \"./data/covid-19_rates/\"\n",
+ "geodata_path = \"./data/geodata/geo_data.csv\"\n"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
+ "jupyter": {
+ "source_hidden": true
+ },
"papermill": {
- "duration": 0.02835,
- "end_time": "2020-03-23T08:23:17.166916",
+ "duration": 0.018129,
+ "end_time": "2020-03-23T22:44:15.619484",
"exception": false,
- "start_time": "2020-03-23T08:23:17.138566",
+ "start_time": "2020-03-23T22:44:15.601355",
"status": "completed"
},
"tags": []
@@ -93,11 +105,14 @@
"cell_type": "code",
"execution_count": 5,
"metadata": {
+ "jupyter": {
+ "source_hidden": true
+ },
"papermill": {
- "duration": 0.209046,
- "end_time": "2020-03-23T08:23:17.397170",
+ "duration": 0.102976,
+ "end_time": "2020-03-23T22:44:15.735330",
"exception": false,
- "start_time": "2020-03-23T08:23:17.188124",
+ "start_time": "2020-03-23T22:44:15.632354",
"status": "completed"
},
"tags": []
@@ -118,10 +133,10 @@
"cell_type": "markdown",
"metadata": {
"papermill": {
- "duration": 0.018688,
- "end_time": "2020-03-23T08:23:17.434967",
+ "duration": 0.009854,
+ "end_time": "2020-03-23T22:44:15.756621",
"exception": false,
- "start_time": "2020-03-23T08:23:17.416279",
+ "start_time": "2020-03-23T22:44:15.746767",
"status": "completed"
},
"tags": []
@@ -138,11 +153,14 @@
"cell_type": "code",
"execution_count": 6,
"metadata": {
+ "jupyter": {
+ "source_hidden": true
+ },
"papermill": {
- "duration": 0.046636,
- "end_time": "2020-03-23T08:23:17.494796",
+ "duration": 0.02681,
+ "end_time": "2020-03-23T22:44:15.793802",
"exception": false,
- "start_time": "2020-03-23T08:23:17.448160",
+ "start_time": "2020-03-23T22:44:15.766992",
"status": "completed"
},
"tags": []
@@ -170,10 +188,10 @@
"cell_type": "markdown",
"metadata": {
"papermill": {
- "duration": 0.01122,
- "end_time": "2020-03-23T08:23:17.527785",
+ "duration": 0.010503,
+ "end_time": "2020-03-23T22:44:15.814498",
"exception": false,
- "start_time": "2020-03-23T08:23:17.516565",
+ "start_time": "2020-03-23T22:44:15.803995",
"status": "completed"
},
"tags": []
@@ -186,11 +204,14 @@
"cell_type": "code",
"execution_count": 7,
"metadata": {
+ "jupyter": {
+ "source_hidden": true
+ },
"papermill": {
- "duration": 0.024109,
- "end_time": "2020-03-23T08:23:17.562009",
+ "duration": 0.017998,
+ "end_time": "2020-03-23T22:44:15.842555",
"exception": false,
- "start_time": "2020-03-23T08:23:17.537900",
+ "start_time": "2020-03-23T22:44:15.824557",
"status": "completed"
},
"tags": []
@@ -204,11 +225,14 @@
"cell_type": "code",
"execution_count": 8,
"metadata": {
+ "jupyter": {
+ "source_hidden": true
+ },
"papermill": {
- "duration": 0.057608,
- "end_time": "2020-03-23T08:23:17.632656",
+ "duration": 0.035197,
+ "end_time": "2020-03-23T22:44:15.891115",
"exception": false,
- "start_time": "2020-03-23T08:23:17.575048",
+ "start_time": "2020-03-23T22:44:15.855918",
"status": "completed"
},
"tags": []
@@ -222,11 +246,14 @@
"cell_type": "code",
"execution_count": 9,
"metadata": {
+ "jupyter": {
+ "source_hidden": true
+ },
"papermill": {
- "duration": 0.167134,
- "end_time": "2020-03-23T08:23:17.823070",
+ "duration": 0.093157,
+ "end_time": "2020-03-23T22:44:15.994479",
"exception": false,
- "start_time": "2020-03-23T08:23:17.655936",
+ "start_time": "2020-03-23T22:44:15.901322",
"status": "completed"
},
"tags": []
@@ -236,10 +263,10 @@
"data": {
"text/html": [
"\n",
- "<div id=\"altair-viz-b1c9a07f93534ca7b0e7039addf0b657\"></div>\n",
+ "<div id=\"altair-viz-4b15a32c7e3f4262b5f3889b4786d9fd\"></div>\n",
"<script type=\"text/javascript\">\n",
" (function(spec, embedOpt){\n",
- " const outputDiv = document.getElementById(\"altair-viz-b1c9a07f93534ca7b0e7039addf0b657\");\n",
+ " const outputDiv = document.getElementById(\"altair-viz-4b15a32c7e3f4262b5f3889b4786d9fd\");\n",
" const paths = {\n",
" \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
" \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
@@ -322,11 +349,14 @@
"cell_type": "code",
"execution_count": 10,
"metadata": {
+ "jupyter": {
+ "source_hidden": true
+ },
"papermill": {
- "duration": 0.108717,
- "end_time": "2020-03-23T08:23:17.950929",
+ "duration": 0.105797,
+ "end_time": "2020-03-23T22:44:16.114645",
"exception": false,
- "start_time": "2020-03-23T08:23:17.842212",
+ "start_time": "2020-03-23T22:44:16.008848",
"status": "completed"
},
"tags": []
@@ -336,10 +366,10 @@
"data": {
"text/html": [
"\n",
- "<div id=\"altair-viz-7f4d5696819f48af9e2a3026f9ea0b6b\"></div>\n",
+ "<div id=\"altair-viz-84db540ff714462a8c23113e18894495\"></div>\n",
"<script type=\"text/javascript\">\n",
" (function(spec, embedOpt){\n",
- " const outputDiv = document.getElementById(\"altair-viz-7f4d5696819f48af9e2a3026f9ea0b6b\");\n",
+ " const outputDiv = document.getElementById(\"altair-viz-84db540ff714462a8c23113e18894495\");\n",
" const paths = {\n",
" \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
" \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
@@ -439,10 +469,10 @@
"cell_type": "markdown",
"metadata": {
"papermill": {
- "duration": 0.013453,
- "end_time": "2020-03-23T08:23:17.987581",
+ "duration": 0.013322,
+ "end_time": "2020-03-23T22:44:16.148918",
"exception": false,
- "start_time": "2020-03-23T08:23:17.974128",
+ "start_time": "2020-03-23T22:44:16.135596",
"status": "completed"
},
"tags": []
@@ -455,11 +485,14 @@
"cell_type": "code",
"execution_count": 11,
"metadata": {
+ "jupyter": {
+ "source_hidden": true
+ },
"papermill": {
- "duration": 0.133644,
- "end_time": "2020-03-23T08:23:18.133548",
+ "duration": 0.070738,
+ "end_time": "2020-03-23T22:44:16.233008",
"exception": false,
- "start_time": "2020-03-23T08:23:17.999904",
+ "start_time": "2020-03-23T22:44:16.162270",
"status": "completed"
},
"tags": []
@@ -477,11 +510,14 @@
"cell_type": "code",
"execution_count": 12,
"metadata": {
+ "jupyter": {
+ "source_hidden": true
+ },
"papermill": {
- "duration": 0.180195,
- "end_time": "2020-03-23T08:23:18.336375",
+ "duration": 0.182654,
+ "end_time": "2020-03-23T22:44:16.442487",
"exception": false,
- "start_time": "2020-03-23T08:23:18.156180",
+ "start_time": "2020-03-23T22:44:16.259833",
"status": "completed"
},
"tags": []
@@ -491,10 +527,10 @@
"data": {
"text/html": [
"\n",
- "<div id=\"altair-viz-0c5baa9eed7149d2a6ab24097c68d925\"></div>\n",
+ "<div id=\"altair-viz-75c10d06694c4ecc89a92c6ac0760081\"></div>\n",
"<script type=\"text/javascript\">\n",
" (function(spec, embedOpt){\n",
- " const outputDiv = document.getElementById(\"altair-viz-0c5baa9eed7149d2a6ab24097c68d925\");\n",
+ " const outputDiv = document.getElementById(\"altair-viz-75c10d06694c4ecc89a92c6ac0760081\");\n",
" const paths = {\n",
" \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
" \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
@@ -587,11 +623,14 @@
"cell_type": "code",
"execution_count": 13,
"metadata": {
+ "jupyter": {
+ "source_hidden": true
+ },
"papermill": {
- "duration": 0.3671,
- "end_time": "2020-03-23T08:23:18.726794",
+ "duration": 0.33824,
+ "end_time": "2020-03-23T22:44:16.804706",
"exception": false,
- "start_time": "2020-03-23T08:23:18.359694",
+ "start_time": "2020-03-23T22:44:16.466466",
"status": "completed"
},
"tags": []
@@ -621,11 +660,14 @@
"cell_type": "code",
"execution_count": 14,
"metadata": {
+ "jupyter": {
+ "source_hidden": true
+ },
"papermill": {
- "duration": 0.056562,
- "end_time": "2020-03-23T08:23:18.819418",
+ "duration": 0.054127,
+ "end_time": "2020-03-23T22:44:16.893122",
"exception": false,
- "start_time": "2020-03-23T08:23:18.762856",
+ "start_time": "2020-03-23T22:44:16.838995",
"status": "completed"
},
"tags": []
@@ -676,11 +718,14 @@
"cell_type": "code",
"execution_count": 15,
"metadata": {
+ "jupyter": {
+ "source_hidden": true
+ },
"papermill": {
- "duration": 0.417622,
- "end_time": "2020-03-23T08:23:19.272334",
+ "duration": 0.408029,
+ "end_time": "2020-03-23T22:44:17.335579",
"exception": false,
- "start_time": "2020-03-23T08:23:18.854712",
+ "start_time": "2020-03-23T22:44:16.927550",
"status": "completed"
},
"tags": []
@@ -690,10 +735,10 @@
"data": {
"text/html": [
"\n",
- "<div id=\"altair-viz-3f760dd3750d4e7a8cfb409f54e5b587\"></div>\n",
+ "<div id=\"altair-viz-7a0b6d7a014e4c97986623a0b632e8ef\"></div>\n",
"<script type=\"text/javascript\">\n",
" (function(spec, embedOpt){\n",
- " const outputDiv = document.getElementById(\"altair-viz-3f760dd3750d4e7a8cfb409f54e5b587\");\n",
+ " const outputDiv = document.getElementById(\"altair-viz-7a0b6d7a014e4c97986623a0b632e8ef\");\n",
" const paths = {\n",
" \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
" \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
@@ -788,11 +833,14 @@
"cell_type": "code",
"execution_count": 16,
"metadata": {
+ "jupyter": {
+ "source_hidden": true
+ },
"papermill": {
- "duration": 0.436264,
- "end_time": "2020-03-23T08:23:19.741535",
+ "duration": 0.41937,
+ "end_time": "2020-03-23T22:44:17.788061",
"exception": false,
- "start_time": "2020-03-23T08:23:19.305271",
+ "start_time": "2020-03-23T22:44:17.368691",
"status": "completed"
},
"tags": []
@@ -802,10 +850,10 @@
"data": {
"text/html": [
"\n",
- "<div id=\"altair-viz-bc8b49b8769a43d5aeacdb9c141732ba\"></div>\n",
+ "<div id=\"altair-viz-149c5820cccd4622b63bc5a3fc01d728\"></div>\n",
"<script type=\"text/javascript\">\n",
" (function(spec, embedOpt){\n",
- " const outputDiv = document.getElementById(\"altair-viz-bc8b49b8769a43d5aeacdb9c141732ba\");\n",
+ " const outputDiv = document.getElementById(\"altair-viz-149c5820cccd4622b63bc5a3fc01d728\");\n",
" const paths = {\n",
" \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
" \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
@@ -896,11 +944,14 @@
"cell_type": "code",
"execution_count": 17,
"metadata": {
+ "jupyter": {
+ "source_hidden": true
+ },
"papermill": {
- "duration": 0.459847,
- "end_time": "2020-03-23T08:23:20.236170",
+ "duration": 0.439979,
+ "end_time": "2020-03-23T22:44:18.262839",
"exception": false,
- "start_time": "2020-03-23T08:23:19.776323",
+ "start_time": "2020-03-23T22:44:17.822860",
"status": "completed"
},
"tags": []
@@ -910,10 +961,10 @@
"data": {
"text/html": [
"\n",
- "<div id=\"altair-viz-e68a80a9ac6a43dd881cee59af1940d7\"></div>\n",
+ "<div id=\"altair-viz-c99ea9676a8f4feba5e92d88678756f8\"></div>\n",
"<script type=\"text/javascript\">\n",
" (function(spec, embedOpt){\n",
- " const outputDiv = document.getElementById(\"altair-viz-e68a80a9ac6a43dd881cee59af1940d7\");\n",
+ " const outputDiv = document.getElementById(\"altair-viz-c99ea9676a8f4feba5e92d88678756f8\");\n",
" const paths = {\n",
" \"vega\": \"https://cdn.jsdelivr.net/npm//vega@5?noext\",\n",
" \"vega-lib\": \"https://cdn.jsdelivr.net/npm//vega-lib?noext\",\n",
@@ -1004,6 +1055,7 @@
}
],
"metadata": {
+ "hide_input": true,
"kernelspec": {
"display_name": "Python 3",
"language": "python",
@@ -1022,20 +1074,20 @@
"version": "3.7.3"
},
"papermill": {
- "duration": 5.260242,
- "end_time": "2020-03-23T08:23:20.607024",
+ "duration": 4.339333,
+ "end_time": "2020-03-23T22:44:18.610153",
"environment_variables": {},
"exception": null,
- "input_path": "/tmp/zlogjitb/notebooks/Dashboard.ipynb",
+ "input_path": "notebooks/Dashboard.ipynb",
"output_path": "runs/Dashboard.run.ipynb",
"parameters": {
- "PAPERMILL_INPUT_PATH": "/tmp/zlogjitb/notebooks/Dashboard.ipynb",
+ "PAPERMILL_INPUT_PATH": "notebooks/Dashboard.ipynb",
"PAPERMILL_OUTPUT_PATH": "runs/Dashboard.run.ipynb",
- "geodata_path": "/tmp/zlogjitb/data/geodata/geo_data.csv",
- "rates_folder": "/tmp/zlogjitb/data/covid-19_rates",
- "ts_folder": "/tmp/zlogjitb/data/covid-19_jhu-csse"
+ "geodata_path": "./data/geodata/geo_data.csv",
+ "rates_folder": "./data/covid-19_rates/",
+ "ts_folder": "./data/covid-19_jhu-csse/"
},
- "start_time": "2020-03-23T08:23:15.346782",
+ "start_time": "2020-03-23T22:44:14.270820",
"version": "1.1.0"
}
},
--
GitLab