.dockerignore

+data
+notebooks
+.renku
+.git

.gitignore

+
+# Created by https://www.gitignore.io/api/macos,python,R,linux,vim,emacs,visualstudiocode,intellij
+# Edit at https://www.gitignore.io/?templates=macos,python,R,linux,vim,emacs,visualstudiocode,intellij
+
+### Emacs ###
+# -*- mode: gitignore; -*-
+*~
+\#*\#
+/.emacs.desktop
+/.emacs.desktop.lock
+*.elc
+auto-save-list
+tramp
+.\#*
+
+# Org-mode
+.org-id-locations
+*_archive
+
+# flymake-mode
+*_flymake.*
+
+# eshell files
+/eshell/history
+/eshell/lastdir
+
+# elpa packages
+/elpa/
+
+# reftex files
+*.rel
+
+# AUCTeX auto folder
+/auto/
+
+# cask packages
+.cask/
+dist/
+
+# Flycheck
+flycheck_*.el
+
+# server auth directory
+/server/
+
+# projectiles files
+.projectile
+
+# directory configuration
+.dir-locals.el
+
+# network security
+/network-security.data
+
+
+### Intellij ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+# JetBrains templates
+**___jb_tmp___
+
+### Intellij Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+.idea/sonarlint
+
+### Linux ###
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+### R ###
+# History files
+.Rhistory
+.Rapp.history
+
+# Session Data files
+.RData
+
+# User-specific files
+.Ruserdata
+
+# Example code in package build process
+*-Ex.R
+
+# Output files from R CMD build
+/*.tar.gz
+
+# Output files from R CMD check
+/*.Rcheck/
+
+# RStudio files
+.Rproj.user/
+
+# produced vignettes
+vignettes/*.html
+vignettes/*.pdf
+
+# OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3
+.httr-oauth
+
+# knitr and R markdown default cache directories
+/*_cache/
+/cache/
+
+# Temporary files created by R markdown
+*.utf8.md
+*.knit.md
+
+### R.Bookdown Stack ###
+# R package: bookdown caching files
+/*_files/
+
+### Vim ###
+# Swap
+[._]*.s[a-v][a-z]
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+
+# Temporary
+.netrwhist
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+
+# End of https://www.gitignore.io/api/macos,python,R,linux,vim,emacs,visualstudiocode,intellij
+
+# Renku
+.renku.lock
+.renku/tmp
+.renku/cache
+
+.vscode

.gitlab-ci.yml

+variables:
+  GIT_STRATEGY: fetch
+  GIT_SSL_NO_VERIFY: "true"
+  GIT_SUBMODULE_STRATEGY: recursive
+  GIT_LFS_SKIP_SMUDGE: 1
+  DOCKER_BUILDKIT: 1
+
+.docker-before-script: &docker_before_script  # This is an anchor
+  before_script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN http://$CI_REGISTRY
+
+.git-before-script: &git_before_script
+  before_script:
+    - git config --global user.name "CR (covid cron)"
+    - git config --global user.email "beepbop@example.com"
+    - url_host=`git remote get-url origin | sed -e "s/https:\/\/gitlab-ci-token:.*@//g"`
+    - cd /tmp/
+    - git clone "https://oauth2:${CI_PUSH_TOKEN}@${url_host}"
+    - cd covid-19-public-data
+    - git lfs install --local
+    - git checkout $CI_COMMIT_BRANCH
+    - PATH=$PATH:~/.local/bin
+
+stages:
+  - build
+  - datasets-update
+  - notebooks-update
+  - finalize-update
+
+image_build:
+  stage: build
+  image: docker:stable
+  except:
+    - /^auto-update.*$/
+  <<: *docker_before_script
+
+  script: |
+    CI_COMMIT_SHA_7=$(echo $CI_COMMIT_SHA | cut -c1-7)
+
+    # export the short SHA for other stages
+    echo "CI_COMMIT_SHA_7=$CI_COMMIT_SHA_7" >> build.env
+
+    # build image
+    docker build --build-arg BUILDKIT_INLINE_CACHE=1 \
+      --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 \
+      --cache-from $CI_REGISTRY_IMAGE .
+
+    # push with commit-sha tag and latest
+    docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7
+    docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 $CI_REGISTRY_IMAGE:latest
+    docker push $CI_REGISTRY_IMAGE:latest
+  artifacts:
+    reports:
+      dotenv: build.env
+
+datasets:
+  stage: datasets-update
+  image:
+    name: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA_7}
+    entrypoint: [""]
+
+  only:
+    - schedules
+  <<: *git_before_script
+  script:
+    - export BRANCH_NAME=auto-update-datasets-${CI_COMMIT_SHA_7}
+    - git checkout -b $BRANCH_NAME
+    # Johns Hopkins
+    - renku dataset update covid-19_jhu-csse
+    - renku dataset add covid-19_jhu-csse --force --source csse_covid_19_data/csse_covid_19_daily_reports https://github.com/CSSEGISandData/COVID-19.git
+
+    # Open Data Zürich
+    - renku dataset update openzh-covid-19
+
+    # Italy
+    - renku dataset update covid-19-italy
+
+    # Spain
+    - renku dataset update covid-19-spain
+
+    # NYT US data
+    - renku dataset update covid-19-us-nyt
+
+    # ECDC
+    - wget -O /tmp/covid-19-ecdc.csv https://opendata.ecdc.europa.eu/covid19/casedistribution/csv
+    - renku dataset add covid-19-ecdc /tmp/covid-19-ecdc.csv --overwrite || true
+
+    # chile data has not been updated for a long time
+    # - renku dataset add covid-19-chile -s data/*.csv  -s data/covid19_chile.rds https://github.com/itoledor/coronavirus.git --force || true
+
+    # Covidtracker no longer collecting data
+    # TODO: transition to federal data sources: https://covidtracking.com/analysis-updates/federal-covid-data-single-stream
+    # - renku dataset add covidtracker https://ocgptweb.azurewebsites.net/CSVDownload -d covidtracker.csv --force || true
+    # - renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json
+    - git push --set-upstream origin $BRANCH_NAME
+
+notebooks:
+  stage: notebooks-update
+  image:
+    name: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA_7}
+    entrypoint: [""]
+  only:
+    refs:
+      - schedules
+  <<: *git_before_script
+  script:
+    - export BRANCH_NAME=auto-update-notebooks-${CI_COMMIT_SHA_7}
+    - git checkout auto-update-datasets-${CI_COMMIT_SHA_7}
+    - git checkout -b $BRANCH_NAME
+    - renku update --all
+    - git push --set-upstream origin $BRANCH_NAME
+
+finalize:
+  stage: finalize-update
+  image:
+    name: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA_7}
+    entrypoint: [""]
+  only:
+    - schedules
+  <<: *git_before_script
+  script: |
+    set -x
+
+    # install jq
+    wget -O ~/.local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64
+    chmod u+x ~/.local/bin/jq
+
+    # push the updated branch to a common name
+    export BRANCH_NAME="auto-update"_$(date -u +"%Y-%m-%d_%H-%M")
+    git checkout auto-update-notebooks-${CI_COMMIT_SHA_7} || git checkout auto-update-datasets-${CI_COMMIT_SHA_7}
+    git rev-parse --abbrev-ref HEAD
+    git checkout -b $BRANCH_NAME
+    git push origin $BRANCH_NAME
+
+    # create MR
+    BODY="{
+      \"id\": ${CI_PROJECT_ID},
+      \"source_branch\": \"${BRANCH_NAME}\",
+      \"target_branch\": \"master\",
+      \"remove_source_branch\": true,
+      \"title\": \"Automatic update - ${BRANCH_NAME}\"
+      }";
+
+    MR_ID=$(curl -s -X POST "https://renkulab.io/gitlab/api/v4/projects/${CI_PROJECT_ID}/merge_requests" \
+      --header "PRIVATE-TOKEN: ${CI_PUSH_TOKEN}" \
+      --header "Content-Type: application/json" \
+      --data "${BODY}" | jq .iid)
+
+    # merge MR
+    RES=$(curl -s -X PUT "https://renkulab.io/gitlab/api/v4/projects/${CI_PROJECT_ID}/merge_requests/${MR_ID}/merge" \
+      --header "PRIVATE-TOKEN: ${CI_PUSH_TOKEN}" \
+      --header "Content-Type: application/json")
+    echo $RES
+
+    # clean up branches
+    git push origin --delete auto-update-notebooks-${CI_COMMIT_SHA_7} || true
+    git push origin --delete auto-update-datasets-${CI_COMMIT_SHA_7} || true

.renku/metadata.yml

+# Dummy file kept for backwards compatibility, does not contain actual version
+'http://schema.org/schemaVersion': '9'
\ No newline at end of file