.gitignore

@@ -378,3 +378,5 @@ tags
 .renku.lock
 .renku/tmp
 .renku/cache
+
+.vscode

.gitlab-ci.yml

-# renku: v1.dev20180723
-
 variables:
   GIT_STRATEGY: fetch
   GIT_SSL_NO_VERIFY: "true"
   GIT_SUBMODULE_STRATEGY: recursive
   GIT_LFS_SKIP_SMUDGE: 1
+  DOCKER_BUILDKIT: 1
+
+.docker-before-script: &docker_before_script  # This is an anchor
+  before_script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN http://$CI_REGISTRY
+
+.git-before-script: &git_before_script
+  before_script:
+    - git config --global user.name "CR (covid cron)"
+    - git config --global user.email "beepbop@example.com"
+    - url_host=`git remote get-url origin | sed -e "s/https:\/\/gitlab-ci-token:.*@//g"`
+    - cd /tmp/
+    - git clone "https://oauth2:${CI_PUSH_TOKEN}@${url_host}"
+    - cd covid-19-public-data
+    - git lfs install --local
+    - git checkout $CI_COMMIT_BRANCH
+    - PATH=$PATH:~/.local/bin
 
 stages:
   - build
+  - datasets-update
+  - notebooks-update
+  - finalize-update
 
 image_build:
   stage: build
   image: docker:stable
-  before_script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN http://$CI_REGISTRY
+  except:
+    - /^auto-update.*$/
+  <<: *docker_before_script
+
+  script: |
+    CI_COMMIT_SHA_7=$(echo $CI_COMMIT_SHA | cut -c1-7)
+
+    # export the short SHA for other stages
+    echo "CI_COMMIT_SHA_7=$CI_COMMIT_SHA_7" >> build.env
+
+    # build image
+    docker build --build-arg BUILDKIT_INLINE_CACHE=1 \
+      --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 \
+      --cache-from $CI_REGISTRY_IMAGE .
+
+    # push with commit-sha tag and latest
+    docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7
+    docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 $CI_REGISTRY_IMAGE:latest
+    docker push $CI_REGISTRY_IMAGE:latest
+  artifacts:
+    reports:
+      dotenv: build.env
+
+datasets:
+  stage: datasets-update
+  image:
+    name: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA_7}
+    entrypoint: [""]
+
+  only:
+    - schedules
+  <<: *git_before_script
+  script:
+    - export BRANCH_NAME=auto-update-datasets-${CI_COMMIT_SHA_7}
+    - git checkout -b $BRANCH_NAME
+    # Johns Hopkins
+    - renku dataset update covid-19_jhu-csse
+    - renku dataset add covid-19_jhu-csse --force --source csse_covid_19_data/csse_covid_19_daily_reports https://github.com/CSSEGISandData/COVID-19.git
+
+    # Open Data Zürich
+    - renku dataset update openzh-covid-19
+
+    # Italy
+    - renku dataset update covid-19-italy
+
+    # Spain
+    - renku dataset update covid-19-spain
+
+    # NYT US data
+    - renku dataset update covid-19-us-nyt
+
+    # ECDC
+    - wget -O /tmp/covid-19-ecdc.csv https://opendata.ecdc.europa.eu/covid19/casedistribution/csv
+    - renku dataset add covid-19-ecdc /tmp/covid-19-ecdc.csv --overwrite || true
+
+    # chile data has not been updated for a long time
+    # - renku dataset add covid-19-chile -s data/*.csv  -s data/covid19_chile.rds https://github.com/itoledor/coronavirus.git --force || true
+
+    # Covidtracker no longer collecting data
+    # TODO: transition to federal data sources: https://covidtracking.com/analysis-updates/federal-covid-data-single-stream
+    # - renku dataset add covidtracker https://ocgptweb.azurewebsites.net/CSVDownload -d covidtracker.csv --force || true
+    # - renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json
+    - git push --set-upstream origin $BRANCH_NAME
+
+notebooks:
+  stage: notebooks-update
+  image:
+    name: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA_7}
+    entrypoint: [""]
+  only:
+    refs:
+      - schedules
+  <<: *git_before_script
   script:
-    - CI_COMMIT_SHA_7=$(echo $CI_COMMIT_SHA | cut -c1-7)
-    - docker build --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 .
-    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7
-  tags:
-    - image-build
+    - export BRANCH_NAME=auto-update-notebooks-${CI_COMMIT_SHA_7}
+    - git checkout auto-update-datasets-${CI_COMMIT_SHA_7}
+    - git checkout -b $BRANCH_NAME
+    - renku update --all
+    - git push --set-upstream origin $BRANCH_NAME
+
+finalize:
+  stage: finalize-update
+  image:
+    name: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA_7}
+    entrypoint: [""]
+  only:
+    - schedules
+  <<: *git_before_script
+  script: |
+    set -x
+
+    # install jq
+    wget -O ~/.local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64
+    chmod u+x ~/.local/bin/jq
+
+    # push the updated branch to a common name
+    export BRANCH_NAME="auto-update"_$(date -u +"%Y-%m-%d_%H-%M")
+    git checkout auto-update-notebooks-${CI_COMMIT_SHA_7} || git checkout auto-update-datasets-${CI_COMMIT_SHA_7}
+    git rev-parse --abbrev-ref HEAD
+    git checkout -b $BRANCH_NAME
+    git push origin $BRANCH_NAME
+
+    # create MR
+    BODY="{
+      \"id\": ${CI_PROJECT_ID},
+      \"source_branch\": \"${BRANCH_NAME}\",
+      \"target_branch\": \"master\",
+      \"remove_source_branch\": true,
+      \"title\": \"Automatic update - ${BRANCH_NAME}\"
+      }";
+
+    MR_ID=$(curl -s -X POST "https://renkulab.io/gitlab/api/v4/projects/${CI_PROJECT_ID}/merge_requests" \
+      --header "PRIVATE-TOKEN: ${CI_PUSH_TOKEN}" \
+      --header "Content-Type: application/json" \
+      --data "${BODY}" | jq .iid)
+
+    # merge MR
+    RES=$(curl -s -X PUT "https://renkulab.io/gitlab/api/v4/projects/${CI_PROJECT_ID}/merge_requests/${MR_ID}/merge" \
+      --header "PRIVATE-TOKEN: ${CI_PUSH_TOKEN}" \
+      --header "Content-Type: application/json")
+    echo $RES
+
+    # clean up branches
+    git push origin --delete auto-update-notebooks-${CI_COMMIT_SHA_7} || true
+    git push origin --delete auto-update-datasets-${CI_COMMIT_SHA_7} || true

.renku/datasets/720e46f5-0f38-48be-86ef-1fcbae258a6f/metadata.yml

-'@context':
-  '@version': 1.1
-  _id: '@id'
-  _project:
-    '@context':
-      '@version': 1.1
-      _id: '@id'
-      created: schema:dateCreated
-      creator:
-        '@context':
-          '@version': 1.1
-          _id: '@id'
-          affiliation: schema:affiliation
-          alternate_name: schema:alternateName
-          email: schema:email
-          label: rdfs:label
-          name: schema:name
-          prov: http://www.w3.org/ns/prov#
-          rdfs: http://www.w3.org/2000/01/rdf-schema#
-          schema: http://schema.org/
-        '@id': schema:creator
-      name: schema:name
-      prov: http://www.w3.org/ns/prov#
-      schema: http://schema.org/
-      updated: schema:dateUpdated
-      version: schema:schemaVersion
-    '@id': schema:isPartOf
-  based_on: schema:isBasedOn
-  created: schema:dateCreated
-  creator:
-    '@context':
-      '@version': 1.1
-      _id: '@id'
-      affiliation: schema:affiliation
-      alternate_name: schema:alternateName
-      email: schema:email
-      label: rdfs:label
-      name: schema:name
-      prov: http://www.w3.org/ns/prov#
-      rdfs: http://www.w3.org/2000/01/rdf-schema#
-      schema: http://schema.org/
-    '@id': schema:creator
-  date_published: schema:datePublished
-  description: schema:description
-  files:
-    '@context':
-      '@version': 1.1
-      _id: '@id'
-      _project:
-        '@context':
-          '@version': 1.1
-          _id: '@id'
-          created: schema:dateCreated
-          creator:
-            '@context':
-              '@version': 1.1
-              _id: '@id'
-              affiliation: schema:affiliation
-              alternate_name: schema:alternateName
-              email: schema:email
-              label: rdfs:label
-              name: schema:name
-              prov: http://www.w3.org/ns/prov#
-              rdfs: http://www.w3.org/2000/01/rdf-schema#
-              schema: http://schema.org/
-            '@id': schema:creator
-          name: schema:name
-          prov: http://www.w3.org/ns/prov#
-          schema: http://schema.org/
-          updated: schema:dateUpdated
-          version: schema:schemaVersion
-        '@id': schema:isPartOf
-      added: schema:dateCreated
-      based_on: schema:isBasedOn
-      creator:
-        '@context':
-          '@version': 1.1
-          _id: '@id'
-          affiliation: schema:affiliation
-          alternate_name: schema:alternateName
-          email: schema:email
-          label: rdfs:label
-          name: schema:name
-          prov: http://www.w3.org/ns/prov#
-          rdfs: http://www.w3.org/2000/01/rdf-schema#
-          schema: http://schema.org/
-        '@id': schema:creator
-      name: schema:name
-      path: prov:atLocation
-      prov: http://www.w3.org/ns/prov#
-      schema: http://schema.org/
-      url: schema:url
-      wfprov: http://purl.org/wf4ever/wfprov#
-    '@id': schema:hasPart
-  identifier: schema:identifier
-  in_language:
-    '@context':
-      '@version': 1.1
-      alternate_name: schema:alternateName
-      name: schema:name
-      schema: http://schema.org/
-    '@id': schema:inLanguage
-  keywords: schema:keywords
-  license: schema:license
-  name: schema:name
-  path: prov:atLocation
-  prov: http://www.w3.org/ns/prov#
-  same_as:
-    '@context':
-      '@version': 1.1
-      _id: '@id'
-      schema: http://schema.org/
-      url: schema:url
-    '@id': schema:sameAs
-  schema: http://schema.org/
-  short_name: schema:alternateName
-  tags:
-    '@context':
-      '@version': 1.1
-      _id: '@id'
-      commit: schema:location
-      created: schema:startDate
-      dataset: schema:about
-      description: schema:description
-      name: schema:name
-      schema: http://schema.org/
-    '@id': schema:subjectOf
-  url: schema:url
-  version: schema:version
-  wfprov: http://purl.org/wf4ever/wfprov#
-'@type':
-- prov:Entity
-- schema:Dataset
-- wfprov:Artifact
-_id: https://dev.renku.ch/datasets/720e46f5-0f38-48be-86ef-1fcbae258a6f
-_label: 720e46f5-0f38-48be-86ef-1fcbae258a6f
-_project:
-  '@type':
-  - prov:Location
-  - schema:Project
-  _id: https://dev.renku.ch/projects/cramakri/covid-19-dashboard
-  created: '2020-03-11T21:43:12.736000+00:00'
-  creator:
-    '@type':
-    - prov:Person
-    - schema:Person
-    _id: mailto:cramakri@ethz.ch
-    affiliation: null
-    alternate_name: null
-    email: cramakri@ethz.ch
-    label: Chandrasekhar Ramakrishnan
-    name: Chandrasekhar Ramakrishnan
-  name: covid-19-dashboard
-  updated: '2020-03-11T21:43:12.736000+00:00'
-  version: '1'
-based_on: null
-created: '2020-03-11T21:59:24.774290+00:00'
-creator:
-- '@type':
-  - prov:Person
-  - schema:Person
-  _id: mailto:cramakri@ethz.ch
-  affiliation: null
-  alternate_name: null
-  email: cramakri@ethz.ch
-  label: Chandrasekhar Ramakrishnan
-  name: Chandrasekhar Ramakrishnan
-date_published: null
-description: null
-files:
-- '@type':
-  - prov:Entity
-  - schema:DigitalDocument
-  - wfprov:Artifact
-  _id: https://dev.renku.ch/blob/f9328ad834b29347e00c73c32bc2a99a91103247/data/worldbank/SP.POP.TOTL.zip
-  _label: data/worldbank/SP.POP.TOTL.zip@f9328ad834b29347e00c73c32bc2a99a91103247
-  _project:
-    '@type':
-    - prov:Location
-    - schema:Project
-    _id: https://dev.renku.ch/projects/cramakri/covid-19-dashboard
-    created: '2020-03-11T21:43:12.736000+00:00'
-    creator:
-      '@type':
-      - prov:Person
-      - schema:Person
-      _id: mailto:cramakri@ethz.ch
-      affiliation: null
-      alternate_name: null
-      email: cramakri@ethz.ch
-      label: Chandrasekhar Ramakrishnan
-      name: Chandrasekhar Ramakrishnan
-    name: covid-19-dashboard
-    updated: '2020-03-11T21:43:12.736000+00:00'
-    version: '1'
-  added: '2020-03-11T22:05:58.843468+00:00'
-  based_on: null
-  creator:
-  - '@type':
-    - prov:Person
-    - schema:Person
-    _id: mailto:cramakri@ethz.ch
-    affiliation: null
-    alternate_name: null
-    email: cramakri@ethz.ch
-    label: Chandrasekhar Ramakrishnan
-    name: Chandrasekhar Ramakrishnan
-  name: SP.POP.TOTL.zip
-  path: data/worldbank/SP.POP.TOTL.zip
-  url: http://api.worldbank.org/v2/en/indicator/SP.POP.TOTL?downloadformat=csv
-identifier: 720e46f5-0f38-48be-86ef-1fcbae258a6f
-in_language: null
-keywords: []
-license: null
-name: worldbank
-path: .renku/datasets/720e46f5-0f38-48be-86ef-1fcbae258a6f
-same_as: null
-short_name: worldbank
-tags: []
-url: https://dev.renku.ch/datasets/720e46f5-0f38-48be-86ef-1fcbae258a6f
-version: null

.renku/datasets/bcd7b8bf-840d-45fd-9912-bb26be595d7f/metadata.yml

-'@context':
-  '@version': 1.1
-  _id: '@id'
-  _project:
-    '@context':
-      '@version': 1.1
-      _id: '@id'
-      created: schema:dateCreated
-      creator:
-        '@context':
-          '@version': 1.1
-          _id: '@id'
-          affiliation: schema:affiliation
-          alternate_name: schema:alternateName
-          email: schema:email
-          label: rdfs:label
-          name: schema:name
-          prov: http://www.w3.org/ns/prov#
-          rdfs: http://www.w3.org/2000/01/rdf-schema#
-          schema: http://schema.org/
-        '@id': schema:creator
-      name: schema:name
-      prov: http://www.w3.org/ns/prov#
-      schema: http://schema.org/
-      updated: schema:dateUpdated
-      version: schema:schemaVersion
-    '@id': schema:isPartOf
-  based_on: schema:isBasedOn
-  created: schema:dateCreated
-  creator:
-    '@context':
-      '@version': 1.1
-      _id: '@id'
-      affiliation: schema:affiliation
-      alternate_name: schema:alternateName
-      email: schema:email
-      label: rdfs:label
-      name: schema:name
-      prov: http://www.w3.org/ns/prov#
-      rdfs: http://www.w3.org/2000/01/rdf-schema#
-      schema: http://schema.org/
-    '@id': schema:creator
-  date_published: schema:datePublished
-  description: schema:description
-  files:
-    '@context':
-      '@version': 1.1
-      _id: '@id'
-      _project:
-        '@context':
-          '@version': 1.1
-          _id: '@id'
-          created: schema:dateCreated
-          creator:
-            '@context':
-              '@version': 1.1
-              _id: '@id'
-              affiliation: schema:affiliation
-              alternate_name: schema:alternateName
-              email: schema:email
-              label: rdfs:label
-              name: schema:name
-              prov: http://www.w3.org/ns/prov#
-              rdfs: http://www.w3.org/2000/01/rdf-schema#
-              schema: http://schema.org/
-            '@id': schema:creator
-          name: schema:name
-          prov: http://www.w3.org/ns/prov#
-          schema: http://schema.org/
-          updated: schema:dateUpdated
-          version: schema:schemaVersion
-        '@id': schema:isPartOf
-      added: schema:dateCreated
-      based_on: schema:isBasedOn
-      creator:
-        '@context':
-          '@version': 1.1
-          _id: '@id'
-          affiliation: schema:affiliation
-          alternate_name: schema:alternateName
-          email: schema:email
-          label: rdfs:label
-          name: schema:name
-          prov: http://www.w3.org/ns/prov#
-          rdfs: http://www.w3.org/2000/01/rdf-schema#
-          schema: http://schema.org/
-        '@id': schema:creator
-      name: schema:name
-      path: prov:atLocation
-      prov: http://www.w3.org/ns/prov#
-      schema: http://schema.org/
-      url: schema:url
-      wfprov: http://purl.org/wf4ever/wfprov#
-    '@id': schema:hasPart
-  identifier: schema:identifier
-  in_language:
-    '@context':
-      '@version': 1.1
-      alternate_name: schema:alternateName
-      name: schema:name
-      schema: http://schema.org/
-    '@id': schema:inLanguage
-  keywords: schema:keywords
-  license: schema:license
-  name: schema:name
-  path: prov:atLocation
-  prov: http://www.w3.org/ns/prov#
-  same_as:
-    '@context':
-      '@version': 1.1
-      _id: '@id'
-      schema: http://schema.org/
-      url: schema:url
-    '@id': schema:sameAs
-  schema: http://schema.org/
-  short_name: schema:alternateName
-  tags:
-    '@context':
-      '@version': 1.1
-      _id: '@id'
-      commit: schema:location
-      created: schema:startDate
-      dataset: schema:about
-      description: schema:description
-      name: schema:name
-      schema: http://schema.org/
-    '@id': schema:subjectOf
-  url: schema:url
-  version: schema:version
-  wfprov: http://purl.org/wf4ever/wfprov#
-'@type':
-- prov:Entity
-- schema:Dataset
-- wfprov:Artifact
-_id: https://dev.renku.ch/datasets/bcd7b8bf-840d-45fd-9912-bb26be595d7f
-_label: bcd7b8bf-840d-45fd-9912-bb26be595d7f
-_project:
-  '@type':
-  - prov:Location
-  - schema:Project
-  _id: https://dev.renku.ch/projects/cramakri/covid-19-dashboard
-  created: '2020-03-11T21:43:12.736000+00:00'
-  creator:
-    '@type':
-    - prov:Person
-    - schema:Person
-    _id: mailto:cramakri@ethz.ch
-    affiliation: null
-    alternate_name: null
-    email: cramakri@ethz.ch
-    label: Chandrasekhar Ramakrishnan
-    name: Chandrasekhar Ramakrishnan
-  name: covid-19-dashboard
-  updated: '2020-03-11T21:43:12.736000+00:00'
-  version: '1'
-based_on: null
-created: '2020-03-14T15:30:35.316239+00:00'
-creator:
-- '@type':
-  - prov:Person
-  - schema:Person
-  _id: mailto:cramakri@ethz.ch
-  affiliation: null
-  alternate_name: null
-  email: cramakri@ethz.ch
-  label: Chandrasekhar Ramakrishnan
-  name: Chandrasekhar Ramakrishnan
-date_published: null
-description: null
-files:
-- '@type':
-  - prov:Entity
-  - schema:DigitalDocument
-  - wfprov:Artifact
-  _id: https://dev.renku.ch/blob/d164d0e430508b65dde128077df316ebdb1cf481/data/worldmap/country_centroids.csv
-  _label: data/worldmap/country_centroids.csv@d164d0e430508b65dde128077df316ebdb1cf481
-  _project:
-    '@type':
-    - prov:Location
-    - schema:Project
-    _id: https://dev.renku.ch/projects/cramakri/covid-19-dashboard
-    created: '2020-03-11T21:43:12.736000+00:00'
-    creator:
-      '@type':
-      - prov:Person
-      - schema:Person
-      _id: mailto:cramakri@ethz.ch
-      affiliation: null
-      alternate_name: null
-      email: cramakri@ethz.ch
-      label: Chandrasekhar Ramakrishnan
-      name: Chandrasekhar Ramakrishnan
-    name: covid-19-dashboard
-    updated: '2020-03-11T21:43:12.736000+00:00'
-    version: '1'
-  added: '2020-03-14T15:43:11.409304+00:00'
-  based_on: null
-  creator:
-  - '@type':
-    - prov:Person
-    - schema:Person
-    _id: mailto:cramakri@ethz.ch
-    affiliation: null
-    alternate_name: null
-    email: cramakri@ethz.ch
-    label: Chandrasekhar Ramakrishnan
-    name: Chandrasekhar Ramakrishnan
-  name: country_centroids.csv
-  path: data/worldmap/country_centroids.csv
-  url: http://worldmap.harvard.edu/download/wfs/34645/csv?outputFormat=csv&service=WFS&request=GetFeature&format_options=charset%3AUTF-8&typename=geonode%3Acountry_centroids_az8&version=1.0.0
-identifier: bcd7b8bf-840d-45fd-9912-bb26be595d7f
-in_language: null
-keywords: []
-license: null
-name: worldmap
-path: .renku/datasets/bcd7b8bf-840d-45fd-9912-bb26be595d7f
-same_as: null
-short_name: worldmap
-tags: []
-url: https://dev.renku.ch/datasets/bcd7b8bf-840d-45fd-9912-bb26be595d7f
-version: null

.renku/metadata.yml

-'@context':
-  created: schema:dateCreated
-  schema: http://schema.org/
-  name: schema:name
-  updated: schema:dateUpdated
-  version: schema:schemaVersion
-'@type': schema:Project
-created: 2020-03-11T21:43:12.736Z
-name: covid-19-dashboard
-updated: 2020-03-11T21:43:12.736Z
-version: '1'
+# Dummy file kept for backwards compatibility, does not contain actual version
+'http://schema.org/schemaVersion': '9'
\ No newline at end of file