Automate complete update pipeline

.gitlab-ci.yml

@@ -3,6 +3,7 @@ variables:
   GIT_SSL_NO_VERIFY: "true"
   GIT_SUBMODULE_STRATEGY: recursive
   GIT_LFS_SKIP_SMUDGE: 1
+  DOCKER_BUILDKIT: 1
 
 .docker-before-script: &docker_before_script  # This is an anchor
   before_script:
@@ -14,7 +15,7 @@ variables:
     - git config --global user.email "beepbop@example.com"
     - url_host=`git remote get-url origin | sed -e "s/https:\/\/gitlab-ci-token:.*@//g"`
     - cd /tmp/
-    - git clone "https://gitlab-ci-token:${CI_PUSH_TOKEN}@${url_host}"
+    - git clone "https://oauth2:${CI_PUSH_TOKEN}@${url_host}"
     - cd covid-19-public-data
     - git lfs install --local
     - PATH=$PATH:~/.local/bin
@@ -33,12 +34,14 @@ image_build:
     - schedules
   <<: *docker_before_script
 
-  script:
-    - CI_COMMIT_SHA_7=$(echo $CI_COMMIT_SHA | cut -c1-7)
-    - docker build --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 .
-    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7
-  tags:
-    - image-build
+  script: |
+    CI_COMMIT_SHA_7=$(echo $CI_COMMIT_SHA | cut -c1-7)
+    docker build --build-arg BUILDKIT_INLINE_CACHE=1 \
+      --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 \
+      --cache-from $CI_REGISTRY_IMAGE .
+    docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7
+    docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 $CI_REGISTRY_IMAGE:latest
+    docker push $CI_REGISTRY_IMAGE:latest
 
 update-build:
   stage: build
@@ -47,16 +50,19 @@ update-build:
     - schedules
   <<: *docker_before_script
 
-  script:
-    - docker build --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update .
-    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update
-  tags:
-    - image-build
+  script: |
+    CI_COMMIT_SHA_7=$(echo $CI_COMMIT_SHA | cut -c1-7)
+    docker build --build-arg BUILDKIT_INLINE_CACHE=1 \
+      --tag $CI_REGISTRY_IMAGE:${CI_COMMIT_SHA_7}-update \
+      --cache-from $CI_REGISTRY_IMAGE .
+    docker push $CI_REGISTRY_IMAGE:${CI_COMMIT_SHA_7}-update
+    docker tag $CI_REGISTRY_IMAGE:${CI_COMMIT_SHA_7}-update ${CI_REGISTRY_IMAGE}:latest
+    docker push $CI_REGISTRY_IMAGE:latest
 
 datasets:
   stage: datasets-update
   image:
-    name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update
+    name: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-update
     entrypoint: [""]
 
   only:
@@ -81,7 +87,7 @@ datasets:
 notebooks:
   stage: notebooks-update
   image:
-    name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update
+    name: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-update
     entrypoint: [""]
   only:
     refs:
@@ -97,17 +103,46 @@ notebooks:
 finalize:
   stage: finalize-update
   image:
-    name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update
+    name: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-update
     entrypoint: [""]
   only:
     - schedules
   <<: *git_before_script
   when: always
-  script:
-    - export BRANCH_NAME="auto-update"_$(date -u +"%Y-%m-%d_%H-%M")
-    - git checkout auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || git checkout auto-update-datasets-${CI_COMMIT_SHORT_SHA}
-    - git rev-parse --abbrev-ref HEAD
-    - git checkout -b $BRANCH_NAME
-    - git push origin --delete auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || true
-    - git push origin --delete auto-update-datasets-${CI_COMMIT_SHORT_SHA} || true
-    - git push origin $BRANCH_NAME
+  script: |
+    set -x
+
+    # install jq
+    wget -O ~/.local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64
+    chmod u+x ~/.local/bin/jq
+
+    # push the updated branch to a common name
+    export BRANCH_NAME="auto-update"_$(date -u +"%Y-%m-%d_%H-%M")
+    git checkout auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || git checkout auto-update-datasets-${CI_COMMIT_SHORT_SHA}
+    git rev-parse --abbrev-ref HEAD
+    git checkout -b $BRANCH_NAME
+    git push origin $BRANCH_NAME
+
+    # create MR
+    BODY="{
+      \"id\": ${CI_PROJECT_ID},
+      \"source_branch\": \"${BRANCH_NAME}\",
+      \"target_branch\": \"master\",
+      \"remove_source_branch\": true,
+      \"title\": \"Automatic update - ${BRANCH_NAME}\"
+      }";
+
+    MR_ID=$(curl -s -X POST "https://renkulab.io/gitlab/api/v4/projects/${CI_PROJECT_ID}/merge_requests" \
+      --header "PRIVATE-TOKEN: ${CI_PUSH_TOKEN}" \
+      --header "Content-Type: application/json" \
+      --data "${BODY}" | jq .iid)
+
+    # merge MR
+    RES=$(curl -s -X PUT "https://renkulab.io/gitlab/api/v4/projects/${CI_PROJECT_ID}/merge_requests/${MR_ID}/merge" \
+      --header "PRIVATE-TOKEN: ${CI_PUSH_TOKEN}" \
+      --header "Content-Type: application/json")
+    echo $RES
+
+    # clean up branches
+    git push origin --delete auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || true
+    git push origin --delete auto-update-datasets-${CI_COMMIT_SHORT_SHA} || true