From ee1739cc3429f7cff483e05db946c3b613768d9f Mon Sep 17 00:00:00 2001
From: Rok Roskar <rokroskar@gmail.com>
Date: Mon, 25 May 2020 17:21:04 +0200
Subject: [PATCH] feat: automate MR on update
---
.gitlab-ci.yml | 82 +++++++++++++++++++++++++++++++++-----------------
Dockerfile | 6 +++-
2 files changed, 59 insertions(+), 29 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c6600b661..198dea327 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,7 +14,7 @@ variables:
- git config --global user.email "beepbop@example.com"
- url_host=`git remote get-url origin | sed -e "s/https:\/\/gitlab-ci-token:.*@//g"`
- cd /tmp/
- - git clone "https://gitlab-ci-token:${CI_PUSH_TOKEN}@${url_host}"
+ - git clone "https://oauth2:${CI_PUSH_TOKEN}@${url_host}"
- cd covid-19-public-data
- git lfs install --local
- PATH=$PATH:~/.local/bin
@@ -43,8 +43,8 @@ image_build:
update-build:
stage: build
image: docker:stable
- only:
- - schedules
+ # only:
+ # - schedules
<<: *docker_before_script
script:
@@ -59,23 +59,23 @@ datasets:
name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update
entrypoint: [""]
- only:
- - schedules
+ # only:
+ # - schedules
<<: *git_before_script
script:
- export BRANCH_NAME=auto-update-datasets-${CI_COMMIT_SHORT_SHA}
- git checkout -b $BRANCH_NAME
- - renku dataset update covid-19_jhu-csse
- - renku dataset add covid-19_jhu-csse --force --source csse_covid_19_data/csse_covid_19_daily_reports https://github.com/CSSEGISandData/COVID-19.git
- - renku dataset update covid-19-tweet-ids
+ # - renku dataset update covid-19_jhu-csse
+ # - renku dataset add covid-19_jhu-csse --force --source csse_covid_19_data/csse_covid_19_daily_reports https://github.com/CSSEGISandData/COVID-19.git
+ # - renku dataset update covid-19-tweet-ids
- renku dataset update openzh-covid-19
- - renku dataset update covid-19-italy
- - renku dataset update covid-19-spain
- - renku dataset update covid-19-us-nyt
- - renku dataset add covid-19-ecdc -d covid-19-ecdc.csv https://opendata.ecdc.europa.eu/covid19/casedistribution/csv --force || true
- - renku dataset add covid-19-chile -s data/*.csv -s data/covid19_chile.rds https://github.com/itoledor/coronavirus.git --force || true
- - renku dataset add covidtracker https://ocgptweb.azurewebsites.net/CSVDownload -d covidtracker.csv --force || true
- - renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json
+ # - renku dataset update covid-19-italy
+ # - renku dataset update covid-19-spain
+ # - renku dataset update covid-19-us-nyt
+ # - renku dataset add covid-19-ecdc -d covid-19-ecdc.csv https://opendata.ecdc.europa.eu/covid19/casedistribution/csv --force || true
+ # - renku dataset add covid-19-chile -s data/*.csv -s data/covid19_chile.rds https://github.com/itoledor/coronavirus.git --force || true
+ # - renku dataset add covidtracker https://ocgptweb.azurewebsites.net/CSVDownload -d covidtracker.csv --force || true
+ # - renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json
- git push --set-upstream origin $BRANCH_NAME
notebooks:
@@ -83,9 +83,9 @@ notebooks:
image:
name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update
entrypoint: [""]
- only:
- refs:
- - schedules
+ # only:
+ # refs:
+ # - schedules
<<: *git_before_script
script:
- export BRANCH_NAME=auto-update-notebooks-${CI_COMMIT_SHORT_SHA}
@@ -99,15 +99,41 @@ finalize:
image:
name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update
entrypoint: [""]
- only:
- - schedules
+ # only:
+ # - schedules
<<: *git_before_script
when: always
- script:
- - export BRANCH_NAME="auto-update"_$(date -u +"%Y-%m-%d_%H-%M")
- - git checkout auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || git checkout auto-update-datasets-${CI_COMMIT_SHORT_SHA}
- - git rev-parse --abbrev-ref HEAD
- - git checkout -b $BRANCH_NAME
- - git push origin --delete auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || true
- - git push origin --delete auto-update-datasets-${CI_COMMIT_SHORT_SHA} || true
- - git push origin $BRANCH_NAME
+ script: |
+ set -x
+ export BRANCH_NAME="auto-update"_$(date -u +"%Y-%m-%d_%H-%M")
+ git checkout auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || git checkout auto-update-datasets-${CI_COMMIT_SHORT_SHA}
+ git rev-parse --abbrev-ref HEAD
+ git checkout -b $BRANCH_NAME
+ git push origin --delete auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || true
+ git push origin --delete auto-update-datasets-${CI_COMMIT_SHORT_SHA} || true
+ git push origin $BRANCH_NAME
+ BODY="{
+ \"id\": ${CI_PROJECT_ID},
+ \"source_branch\": \"${BRANCH_NAME}\",
+ \"target_branch\": \"test-MR\",
+ \"remove_source_branch\": true,
+ \"title\": \"Automatic update - ${BRANCH_NAME}\"
+ }";
+ MR_ID=$(curl -s -X POST "https://renkulab.io/gitlab/api/v4/projects/${CI_PROJECT_ID}/merge_requests" \
+ --header "PRIVATE-TOKEN: ${CI_PUSH_TOKEN}" \
+ --header "Content-Type: application/json" \
+ --data "${BODY}" | jq .iid)
+ BODY="{
+ \"id\": ${CI_PROJECT_ID},
+ \"iid\": ${MR_ID},
+ \"should_remove_source_branch\": true,
+ \"merge_when_pipeline_succeeds\": true.
+ \"target_branch\": \"test-MR\"
+ }";
+ RES=$(curl -s -X PUT "https://renkulab.io/gitlab/api/v4/projects/${CI_PROJECT_ID}/merge_requests/merge" \
+ --header "PRIVATE-TOKEN: ${CI_PUSH_TOKEN}" \
+ --header "Content-Type: application/json" \
+ --data "${BODY}")
+ echo $RES
+
+
diff --git a/Dockerfile b/Dockerfile
index 6e0e13c98..dcc53e6c4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -27,7 +27,7 @@ RUN conda env update -q -f /tmp/environment.yml && \
jupyter labextension install --no-build @jupyter-widgets/jupyterlab-manager && \
jupyter lab build && \
conda clean -y --all && \
- conda env export -n "root"
+ conda env export -n "root"
# install our utilities package
@@ -36,3 +36,7 @@ RUN ln -s /code/src
RUN pip install -e src/covid-19/covid_19_utils
RUN pipx install --force --pip-args="--pre" "renku==0.10.4.dev13"
+
+RUN wget -O ~/.local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 && \
+ chmod u+x ~/.local/bin/jq
+
--
GitLab