From 42e1ef38516fe11d1ab1a634fb674e88a2cbf480 Mon Sep 17 00:00:00 2001
From: Rok Roskar <rokroskar@gmail.com>
Date: Thu, 7 May 2020 00:42:47 +0200
Subject: [PATCH] build: split update pipeline into two stages

---
 .gitlab-ci.yml | 89 ++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 64 insertions(+), 25 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0feffff9c..c6600b661 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,23 +1,38 @@
-# renku: v1.dev20180723
-
 variables:
   GIT_STRATEGY: fetch
   GIT_SSL_NO_VERIFY: "true"
   GIT_SUBMODULE_STRATEGY: recursive
   GIT_LFS_SKIP_SMUDGE: 1
 
+.docker-before-script: &docker_before_script  # This is an anchor
+  before_script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN http://$CI_REGISTRY
+
+.git-before-script: &git_before_script
+  before_script:
+    - git config --global user.name "CR (covid cron)"
+    - git config --global user.email "beepbop@example.com"
+    - url_host=`git remote get-url origin | sed -e "s/https:\/\/gitlab-ci-token:.*@//g"`
+    - cd /tmp/
+    - git clone "https://gitlab-ci-token:${CI_PUSH_TOKEN}@${url_host}"
+    - cd covid-19-public-data
+    - git lfs install --local
+    - PATH=$PATH:~/.local/bin
+
 stages:
   - build
-  - periodic
+  - datasets-update
+  - notebooks-update
+  - finalize-update
 
 image_build:
   stage: build
   image: docker:stable
   except:
-    refs:
-      - schedules
-  before_script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN http://$CI_REGISTRY
+    - /^auto-update.*$/
+    - schedules
+  <<: *docker_before_script
+
   script:
     - CI_COMMIT_SHA_7=$(echo $CI_COMMIT_SHA | cut -c1-7)
     - docker build --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 .
@@ -25,39 +40,30 @@ image_build:
   tags:
     - image-build
 
-update_build:
+update-build:
   stage: build
   image: docker:stable
   only:
-    refs:
-      - schedules
-  before_script:
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN http://$CI_REGISTRY
+    - schedules
+  <<: *docker_before_script
+
   script:
     - docker build --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update .
     - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update
   tags:
     - image-build
 
-update:
-  stage: periodic
+datasets:
+  stage: datasets-update
   image:
     name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update
     entrypoint: [""]
 
   only:
-    refs:
-      - schedules
+    - schedules
+  <<: *git_before_script
   script:
-    - git config --global user.name "CR (covid cron)"
-    - git config --global user.email "cramakri+covid-cron@ethz.ch"
-    - url_host=`git remote get-url origin | sed -e "s/https:\/\/gitlab-ci-token:.*@//g"`
-    - cd /tmp/
-    - git clone "https://gitlab-ci-token:${CI_TAG_UPLOAD_TOKEN}@${url_host}"
-    - cd covid-19-public-data
-    - git lfs install --local
-    - export BRANCH_NAME="update"_$(date -u +"%Y-%m-%d_%H-%M")
-    - PATH=$PATH:~/.local/bin
+    - export BRANCH_NAME=auto-update-datasets-${CI_COMMIT_SHORT_SHA}
     - git checkout -b $BRANCH_NAME
     - renku dataset update covid-19_jhu-csse
     - renku dataset add covid-19_jhu-csse --force --source csse_covid_19_data/csse_covid_19_daily_reports https://github.com/CSSEGISandData/COVID-19.git
@@ -70,5 +76,38 @@ update:
     - renku dataset add covid-19-chile -s data/*.csv  -s data/covid19_chile.rds https://github.com/itoledor/coronavirus.git --force || true
     - renku dataset add covidtracker https://ocgptweb.azurewebsites.net/CSVDownload -d covidtracker.csv --force || true
     - renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json
+    - git push --set-upstream origin $BRANCH_NAME
+
+notebooks:
+  stage: notebooks-update
+  image:
+    name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update
+    entrypoint: [""]
+  only:
+    refs:
+      - schedules
+  <<: *git_before_script
+  script:
+    - export BRANCH_NAME=auto-update-notebooks-${CI_COMMIT_SHORT_SHA}
+    - git checkout auto-update-datasets-${CI_COMMIT_SHORT_SHA}
+    - git checkout -b $BRANCH_NAME
     - renku update --with-siblings
     - git push --set-upstream origin $BRANCH_NAME
+
+finalize:
+  stage: finalize-update
+  image:
+    name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update
+    entrypoint: [""]
+  only:
+    - schedules
+  <<: *git_before_script
+  when: always
+  script:
+    - export BRANCH_NAME="auto-update"_$(date -u +"%Y-%m-%d_%H-%M")
+    - git checkout auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || git checkout auto-update-datasets-${CI_COMMIT_SHORT_SHA}
+    - git rev-parse --abbrev-ref HEAD
+    - git checkout -b $BRANCH_NAME
+    - git push origin --delete auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || true
+    - git push origin --delete auto-update-datasets-${CI_COMMIT_SHORT_SHA} || true
+    - git push origin $BRANCH_NAME
-- 
GitLab