build: periodic update of data sources

.gitlab-ci.yml

@@ -8,10 +8,14 @@ variables:
 
 stages:
   - build
+  - periodic
 
 image_build:
   stage: build
   image: docker:stable
+  except:
+    refs:
+      - schedules
   before_script:
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN http://$CI_REGISTRY
   script:
@@ -20,3 +24,43 @@ image_build:
     - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7
   tags:
     - image-build
+
+update_build:
+  stage: build
+  image: docker:stable
+  only:
+    refs:
+      - schedules
+  before_script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN http://$CI_REGISTRY
+  script:
+    - docker build --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA .
+    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
+  tags:
+    - image-build
+
+update:
+  stage: periodic
+  image:
+    name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
+    entrypoint: [""]
+
+  only:
+    refs:
+      - schedules
+  script:
+    - git config --global user.name "CR (covid cron)"
+    - git config --global user.email "cramakri+covid-cron@ethz.ch"
+    - url_host=`git remote get-url origin | sed -e "s/https:\/\/gitlab-ci-token:.*@//g"`
+    - cd /tmp/
+    - git clone "https://gitlab-ci-token:${CI_TAG_UPLOAD_TOKEN}@${url_host}"
+    - cd covid-19-public-data
+    - git lfs install --local
+    - export BRANCH_NAME="update"_$(date -u +"%Y-%m-%d_%H-%M")
+    - PATH=$PATH:~/.local/bin
+    - git checkout -b $BRANCH_NAME
+    - renku dataset update covid-19_jhu-csse  # add when ds updated: - renku dataset update openzh-covid-19 - renku dataset update covid-19-italy
+    - renku dataset update covid-19-tweet-ids
+    - renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json
+    - renku update --with-siblings
+    - git push --set-upstream origin $BRANCH_NAME