build: use registry for caching

.gitlab-ci.yml

@@ -3,6 +3,7 @@ variables:
   GIT_SSL_NO_VERIFY: "true"
   GIT_SUBMODULE_STRATEGY: recursive
   GIT_LFS_SKIP_SMUDGE: 1
+  DOCKER_BUILDKIT: 1
 
 .docker-before-script: &docker_before_script  # This is an anchor
   before_script:
@@ -33,59 +34,64 @@ image_build:
     - schedules
   <<: *docker_before_script
 
-  script:
-    - CI_COMMIT_SHA_7=$(echo $CI_COMMIT_SHA | cut -c1-7)
-    - docker build --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 .
-    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7
-  tags:
-    - image-build
+  script: |
+    CI_COMMIT_SHA_7=$(echo $CI_COMMIT_SHA | cut -c1-7)
+    docker build --build-arg BUILDKIT_INLINE_CACHE=1 \
+      --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 \
+      --cache-from $CI_REGISTRY_IMAGE .
+    docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7
+    docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 $CI_REGISTRY_IMAGE:latest
+    docker push $CI_REGISTRY_IMAGE:latest
 
 update-build:
   stage: build
   image: docker:stable
-  # only:
-  #   - schedules
+  only:
+    - schedules
   <<: *docker_before_script
 
-  script:
-    - docker build --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update .
-    - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update
-  tags:
-    - image-build
+  script: |
+    CI_COMMIT_SHA_7=$(echo $CI_COMMIT_SHA | cut -c1-7)
+    docker build --build-arg BUILDKIT_INLINE_CACHE=1 \
+      --tag $CI_REGISTRY_IMAGE:${CI_COMMIT_SHA_7}-update \
+      --cache-from $CI_REGISTRY_IMAGE .
+    docker push $CI_REGISTRY_IMAGE:${CI_COMMIT_SHA_7}-update
+    docker tag $CI_REGISTRY_IMAGE:${CI_COMMIT_SHA_7}-update ${CI_REGISTRY_IMAGE}:latest
+    docker push $CI_REGISTRY_IMAGE:latest
 
 datasets:
   stage: datasets-update
   image:
-    name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update
+    name: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-update
     entrypoint: [""]
 
-  # only:
-  #   - schedules
+  only:
+    - schedules
   <<: *git_before_script
   script:
     - export BRANCH_NAME=auto-update-datasets-${CI_COMMIT_SHORT_SHA}
     - git checkout -b $BRANCH_NAME
-    # - renku dataset update covid-19_jhu-csse
-    # - renku dataset add covid-19_jhu-csse --force --source csse_covid_19_data/csse_covid_19_daily_reports https://github.com/CSSEGISandData/COVID-19.git
-    # - renku dataset update covid-19-tweet-ids
+    - renku dataset update covid-19_jhu-csse
+    - renku dataset add covid-19_jhu-csse --force --source csse_covid_19_data/csse_covid_19_daily_reports https://github.com/CSSEGISandData/COVID-19.git
+    - renku dataset update covid-19-tweet-ids
     - renku dataset update openzh-covid-19
-    # - renku dataset update covid-19-italy
-    # - renku dataset update covid-19-spain
-    # - renku dataset update covid-19-us-nyt
-    # - renku dataset add covid-19-ecdc -d covid-19-ecdc.csv https://opendata.ecdc.europa.eu/covid19/casedistribution/csv --force || true
-    # - renku dataset add covid-19-chile -s data/*.csv  -s data/covid19_chile.rds https://github.com/itoledor/coronavirus.git --force || true
-    # - renku dataset add covidtracker https://ocgptweb.azurewebsites.net/CSVDownload -d covidtracker.csv --force || true
-    # - renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json
+    - renku dataset update covid-19-italy
+    - renku dataset update covid-19-spain
+    - renku dataset update covid-19-us-nyt
+    - renku dataset add covid-19-ecdc -d covid-19-ecdc.csv https://opendata.ecdc.europa.eu/covid19/casedistribution/csv --force || true
+    - renku dataset add covid-19-chile -s data/*.csv  -s data/covid19_chile.rds https://github.com/itoledor/coronavirus.git --force || true
+    - renku dataset add covidtracker https://ocgptweb.azurewebsites.net/CSVDownload -d covidtracker.csv --force || true
+    - renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json
     - git push --set-upstream origin $BRANCH_NAME
 
 notebooks:
   stage: notebooks-update
   image:
-    name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update
+    name: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-update
     entrypoint: [""]
-  # only:
-  #   refs:
-  #     - schedules
+  only:
+    refs:
+      - schedules
   <<: *git_before_script
   script:
     - export BRANCH_NAME=auto-update-notebooks-${CI_COMMIT_SHORT_SHA}
@@ -97,43 +103,46 @@ notebooks:
 finalize:
   stage: finalize-update
   image:
-    name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update
+    name: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-update
     entrypoint: [""]
-  # only:
-  #   - schedules
+  only:
+    - schedules
   <<: *git_before_script
   when: always
   script: |
     set -x
+
+    # install jq
+    wget -O ~/.local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64
+    chmod u+x ~/.local/bin/jq
+
+    # push the updated branch to a common name
     export BRANCH_NAME="auto-update"_$(date -u +"%Y-%m-%d_%H-%M")
     git checkout auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || git checkout auto-update-datasets-${CI_COMMIT_SHORT_SHA}
     git rev-parse --abbrev-ref HEAD
     git checkout -b $BRANCH_NAME
-    git push origin --delete auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || true
-    git push origin --delete auto-update-datasets-${CI_COMMIT_SHORT_SHA} || true
     git push origin $BRANCH_NAME
+
+    # create MR
     BODY="{
       \"id\": ${CI_PROJECT_ID},
       \"source_branch\": \"${BRANCH_NAME}\",
-      \"target_branch\": \"test-MR\",
+      \"target_branch\": \"master\",
       \"remove_source_branch\": true,
       \"title\": \"Automatic update - ${BRANCH_NAME}\"
       }";
+
     MR_ID=$(curl -s -X POST "https://renkulab.io/gitlab/api/v4/projects/${CI_PROJECT_ID}/merge_requests" \
       --header "PRIVATE-TOKEN: ${CI_PUSH_TOKEN}" \
       --header "Content-Type: application/json" \
       --data "${BODY}" | jq .iid)
-    BODY="{
-      \"id\": ${CI_PROJECT_ID},
-      \"iid\": ${MR_ID},
-      \"should_remove_source_branch\": true,
-      \"merge_when_pipeline_succeeds\": true.
-      \"target_branch\": \"test-MR\"
-      }";
-    RES=$(curl -s -X PUT "https://renkulab.io/gitlab/api/v4/projects/${CI_PROJECT_ID}/merge_requests/merge" \
+
+    # merge MR
+    RES=$(curl -s -X PUT "https://renkulab.io/gitlab/api/v4/projects/${CI_PROJECT_ID}/merge_requests/${MR_ID}/merge" \
       --header "PRIVATE-TOKEN: ${CI_PUSH_TOKEN}" \
-      --header "Content-Type: application/json" \
-      --data "${BODY}")
+      --header "Content-Type: application/json")
     echo $RES
 
-
+    # clean up branches
+    git push origin --delete auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || true
+    git push origin --delete auto-update-datasets-${CI_COMMIT_SHORT_SHA} || true

Dockerfile

@@ -35,8 +35,7 @@ COPY --chown=rstudio:rstudio src /code/src
 RUN ln -s /code/src
 RUN pip install -e src/covid-19/covid_19_utils
 
-RUN pipx install --force --pip-args="--pre" "renku==0.10.4.dev13"
+RUN pipx install --force "renku==0.10.4"
 
 RUN wget -O ~/.local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 && \
     chmod u+x ~/.local/bin/jq
-