diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 198dea327930b41c4c62cf846ba5c08f57319ab1..f1aa23a87b77a1560358a8d8c56d1bf78a803c9b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ variables: GIT_SSL_NO_VERIFY: "true" GIT_SUBMODULE_STRATEGY: recursive GIT_LFS_SKIP_SMUDGE: 1 + DOCKER_BUILDKIT: 1 .docker-before-script: &docker_before_script # This is an anchor before_script: @@ -33,59 +34,64 @@ image_build: - schedules <<: *docker_before_script - script: - - CI_COMMIT_SHA_7=$(echo $CI_COMMIT_SHA | cut -c1-7) - - docker build --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 . - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 - tags: - - image-build + script: | + CI_COMMIT_SHA_7=$(echo $CI_COMMIT_SHA | cut -c1-7) + docker build --build-arg BUILDKIT_INLINE_CACHE=1 \ + --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 \ + --cache-from $CI_REGISTRY_IMAGE . + docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 + docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 $CI_REGISTRY_IMAGE:latest + docker push $CI_REGISTRY_IMAGE:latest update-build: stage: build image: docker:stable - # only: - # - schedules + only: + - schedules <<: *docker_before_script - script: - - docker build --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update . - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update - tags: - - image-build + script: | + CI_COMMIT_SHA_7=$(echo $CI_COMMIT_SHA | cut -c1-7) + docker build --build-arg BUILDKIT_INLINE_CACHE=1 \ + --tag $CI_REGISTRY_IMAGE:${CI_COMMIT_SHA_7}-update \ + --cache-from $CI_REGISTRY_IMAGE . + docker push $CI_REGISTRY_IMAGE:${CI_COMMIT_SHA_7}-update + docker tag $CI_REGISTRY_IMAGE:${CI_COMMIT_SHA_7}-update ${CI_REGISTRY_IMAGE}:latest + docker push $CI_REGISTRY_IMAGE:latest datasets: stage: datasets-update image: - name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update + name: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-update entrypoint: [""] - # only: - # - schedules + only: + - schedules <<: *git_before_script script: - export BRANCH_NAME=auto-update-datasets-${CI_COMMIT_SHORT_SHA} - git checkout -b $BRANCH_NAME - # - renku dataset update covid-19_jhu-csse - # - renku dataset add covid-19_jhu-csse --force --source csse_covid_19_data/csse_covid_19_daily_reports https://github.com/CSSEGISandData/COVID-19.git - # - renku dataset update covid-19-tweet-ids + - renku dataset update covid-19_jhu-csse + - renku dataset add covid-19_jhu-csse --force --source csse_covid_19_data/csse_covid_19_daily_reports https://github.com/CSSEGISandData/COVID-19.git + - renku dataset update covid-19-tweet-ids - renku dataset update openzh-covid-19 - # - renku dataset update covid-19-italy - # - renku dataset update covid-19-spain - # - renku dataset update covid-19-us-nyt - # - renku dataset add covid-19-ecdc -d covid-19-ecdc.csv https://opendata.ecdc.europa.eu/covid19/casedistribution/csv --force || true - # - renku dataset add covid-19-chile -s data/*.csv -s data/covid19_chile.rds https://github.com/itoledor/coronavirus.git --force || true - # - renku dataset add covidtracker https://ocgptweb.azurewebsites.net/CSVDownload -d covidtracker.csv --force || true - # - renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json + - renku dataset update covid-19-italy + - renku dataset update covid-19-spain + - renku dataset update covid-19-us-nyt + - renku dataset add covid-19-ecdc -d covid-19-ecdc.csv https://opendata.ecdc.europa.eu/covid19/casedistribution/csv --force || true + - renku dataset add covid-19-chile -s data/*.csv -s data/covid19_chile.rds https://github.com/itoledor/coronavirus.git --force || true + - renku dataset add covidtracker https://ocgptweb.azurewebsites.net/CSVDownload -d covidtracker.csv --force || true + - renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json - git push --set-upstream origin $BRANCH_NAME notebooks: stage: notebooks-update image: - name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update + name: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-update entrypoint: [""] - # only: - # refs: - # - schedules + only: + refs: + - schedules <<: *git_before_script script: - export BRANCH_NAME=auto-update-notebooks-${CI_COMMIT_SHORT_SHA} @@ -97,43 +103,46 @@ notebooks: finalize: stage: finalize-update image: - name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update + name: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-update entrypoint: [""] - # only: - # - schedules + only: + - schedules <<: *git_before_script when: always script: | set -x + + # install jq + wget -O ~/.local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 + chmod u+x ~/.local/bin/jq + + # push the updated branch to a common name export BRANCH_NAME="auto-update"_$(date -u +"%Y-%m-%d_%H-%M") git checkout auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || git checkout auto-update-datasets-${CI_COMMIT_SHORT_SHA} git rev-parse --abbrev-ref HEAD git checkout -b $BRANCH_NAME - git push origin --delete auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || true - git push origin --delete auto-update-datasets-${CI_COMMIT_SHORT_SHA} || true git push origin $BRANCH_NAME + + # create MR BODY="{ \"id\": ${CI_PROJECT_ID}, \"source_branch\": \"${BRANCH_NAME}\", - \"target_branch\": \"test-MR\", + \"target_branch\": \"master\", \"remove_source_branch\": true, \"title\": \"Automatic update - ${BRANCH_NAME}\" }"; + MR_ID=$(curl -s -X POST "https://renkulab.io/gitlab/api/v4/projects/${CI_PROJECT_ID}/merge_requests" \ --header "PRIVATE-TOKEN: ${CI_PUSH_TOKEN}" \ --header "Content-Type: application/json" \ --data "${BODY}" | jq .iid) - BODY="{ - \"id\": ${CI_PROJECT_ID}, - \"iid\": ${MR_ID}, - \"should_remove_source_branch\": true, - \"merge_when_pipeline_succeeds\": true. - \"target_branch\": \"test-MR\" - }"; - RES=$(curl -s -X PUT "https://renkulab.io/gitlab/api/v4/projects/${CI_PROJECT_ID}/merge_requests/merge" \ + + # merge MR + RES=$(curl -s -X PUT "https://renkulab.io/gitlab/api/v4/projects/${CI_PROJECT_ID}/merge_requests/${MR_ID}/merge" \ --header "PRIVATE-TOKEN: ${CI_PUSH_TOKEN}" \ - --header "Content-Type: application/json" \ - --data "${BODY}") + --header "Content-Type: application/json") echo $RES - + # clean up branches + git push origin --delete auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || true + git push origin --delete auto-update-datasets-${CI_COMMIT_SHORT_SHA} || true diff --git a/Dockerfile b/Dockerfile index dcc53e6c43e1242600fdd4a25645d6e393bf6d30..d3050209e9dcf1d6e687b858c73e680b19cfb3d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,8 +35,7 @@ COPY --chown=rstudio:rstudio src /code/src RUN ln -s /code/src RUN pip install -e src/covid-19/covid_19_utils -RUN pipx install --force --pip-args="--pre" "renku==0.10.4.dev13" +RUN pipx install --force "renku==0.10.4" RUN wget -O ~/.local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 && \ chmod u+x ~/.local/bin/jq -