From 26654930558d7aed003621767fa0e0c5a92a6e95 Mon Sep 17 00:00:00 2001
From: Chandrasekhar Ramakrishnan <cramakri@ethz.ch>
Date: Thu, 19 Mar 2020 16:10:21 +0100
Subject: [PATCH] build: periodic update of data sources
---
.gitlab-ci.yml | 44 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 44 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5fab0508..94ac1cb7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -8,10 +8,14 @@ variables:
stages:
- build
+ - periodic
image_build:
stage: build
image: docker:stable
+ except:
+ refs:
+ - schedules
before_script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN http://$CI_REGISTRY
script:
@@ -20,3 +24,43 @@ image_build:
- docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7
tags:
- image-build
+
+update_build:
+ stage: build
+ image: docker:stable
+ only:
+ refs:
+ - schedules
+ before_script:
+ - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN http://$CI_REGISTRY
+ script:
+ - docker build --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA .
+ - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
+ tags:
+ - image-build
+
+update:
+ stage: periodic
+ image:
+ name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
+ entrypoint: [""]
+
+ only:
+ refs:
+ - schedules
+ script:
+ - git config --global user.name "CR (covid cron)"
+ - git config --global user.email "cramakri+covid-cron@ethz.ch"
+ - url_host=`git remote get-url origin | sed -e "s/https:\/\/gitlab-ci-token:.*@//g"`
+ - cd /tmp/
+ - git clone "https://gitlab-ci-token:${CI_TAG_UPLOAD_TOKEN}@${url_host}"
+ - cd covid-19-public-data
+ - git lfs install --local
+ - export BRANCH_NAME="update"_$(date -u +"%Y-%m-%d_%H-%M")
+ - PATH=$PATH:~/.local/bin
+ - git checkout -b $BRANCH_NAME
+ - renku dataset update covid-19_jhu-csse # add when ds updated: - renku dataset update openzh-covid-19 - renku dataset update covid-19-italy
+ - renku dataset update covid-19-tweet-ids
+ - renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json
+ - renku update --with-siblings
+ - git push --set-upstream origin $BRANCH_NAME
--
GitLab