feat: automate MR on update

.gitlab-ci.yml

@@ -14,7 +14,7 @@ variables:
     - git config --global user.email "beepbop@example.com"
     - url_host=`git remote get-url origin | sed -e "s/https:\/\/gitlab-ci-token:.*@//g"`
     - cd /tmp/
-    - git clone "https://gitlab-ci-token:${CI_PUSH_TOKEN}@${url_host}"
+    - git clone "https://oauth2:${CI_PUSH_TOKEN}@${url_host}"
     - cd covid-19-public-data
     - git lfs install --local
     - PATH=$PATH:~/.local/bin
@@ -43,8 +43,8 @@ image_build:
 update-build:
   stage: build
   image: docker:stable
-  only:
-    - schedules
+  # only:
+  #   - schedules
   <<: *docker_before_script
 
   script:
@@ -59,23 +59,23 @@ datasets:
     name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update
     entrypoint: [""]
 
-  only:
-    - schedules
+  # only:
+  #   - schedules
   <<: *git_before_script
   script:
     - export BRANCH_NAME=auto-update-datasets-${CI_COMMIT_SHORT_SHA}
     - git checkout -b $BRANCH_NAME
-    - renku dataset update covid-19_jhu-csse
-    - renku dataset add covid-19_jhu-csse --force --source csse_covid_19_data/csse_covid_19_daily_reports https://github.com/CSSEGISandData/COVID-19.git
-    - renku dataset update covid-19-tweet-ids
+    # - renku dataset update covid-19_jhu-csse
+    # - renku dataset add covid-19_jhu-csse --force --source csse_covid_19_data/csse_covid_19_daily_reports https://github.com/CSSEGISandData/COVID-19.git
+    # - renku dataset update covid-19-tweet-ids
     - renku dataset update openzh-covid-19
-    - renku dataset update covid-19-italy
-    - renku dataset update covid-19-spain
-    - renku dataset update covid-19-us-nyt
-    - renku dataset add covid-19-ecdc -d covid-19-ecdc.csv https://opendata.ecdc.europa.eu/covid19/casedistribution/csv --force || true
-    - renku dataset add covid-19-chile -s data/*.csv  -s data/covid19_chile.rds https://github.com/itoledor/coronavirus.git --force || true
-    - renku dataset add covidtracker https://ocgptweb.azurewebsites.net/CSVDownload -d covidtracker.csv --force || true
-    - renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json
+    # - renku dataset update covid-19-italy
+    # - renku dataset update covid-19-spain
+    # - renku dataset update covid-19-us-nyt
+    # - renku dataset add covid-19-ecdc -d covid-19-ecdc.csv https://opendata.ecdc.europa.eu/covid19/casedistribution/csv --force || true
+    # - renku dataset add covid-19-chile -s data/*.csv  -s data/covid19_chile.rds https://github.com/itoledor/coronavirus.git --force || true
+    # - renku dataset add covidtracker https://ocgptweb.azurewebsites.net/CSVDownload -d covidtracker.csv --force || true
+    # - renku rerun data/covidtracking/states-metadata.json data/covidtracking/states-daily.json
     - git push --set-upstream origin $BRANCH_NAME
 
 notebooks:
@@ -83,9 +83,9 @@ notebooks:
   image:
     name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update
     entrypoint: [""]
-  only:
-    refs:
-      - schedules
+  # only:
+  #   refs:
+  #     - schedules
   <<: *git_before_script
   script:
     - export BRANCH_NAME=auto-update-notebooks-${CI_COMMIT_SHORT_SHA}
@@ -99,15 +99,41 @@ finalize:
   image:
     name: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA-update
     entrypoint: [""]
-  only:
-    - schedules
+  # only:
+  #   - schedules
   <<: *git_before_script
   when: always
-  script:
-    - export BRANCH_NAME="auto-update"_$(date -u +"%Y-%m-%d_%H-%M")
-    - git checkout auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || git checkout auto-update-datasets-${CI_COMMIT_SHORT_SHA}
-    - git rev-parse --abbrev-ref HEAD
-    - git checkout -b $BRANCH_NAME
-    - git push origin --delete auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || true
-    - git push origin --delete auto-update-datasets-${CI_COMMIT_SHORT_SHA} || true
-    - git push origin $BRANCH_NAME
+  script: |
+    set -x
+    export BRANCH_NAME="auto-update"_$(date -u +"%Y-%m-%d_%H-%M")
+    git checkout auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || git checkout auto-update-datasets-${CI_COMMIT_SHORT_SHA}
+    git rev-parse --abbrev-ref HEAD
+    git checkout -b $BRANCH_NAME
+    git push origin --delete auto-update-notebooks-${CI_COMMIT_SHORT_SHA} || true
+    git push origin --delete auto-update-datasets-${CI_COMMIT_SHORT_SHA} || true
+    git push origin $BRANCH_NAME
+    BODY="{
+      \"id\": ${CI_PROJECT_ID},
+      \"source_branch\": \"${BRANCH_NAME}\",
+      \"target_branch\": \"test-MR\",
+      \"remove_source_branch\": true,
+      \"title\": \"Automatic update - ${BRANCH_NAME}\"
+      }";
+    MR_ID=$(curl -s -X POST "https://renkulab.io/gitlab/api/v4/projects/${CI_PROJECT_ID}/merge_requests" \
+      --header "PRIVATE-TOKEN: ${CI_PUSH_TOKEN}" \
+      --header "Content-Type: application/json" \
+      --data "${BODY}" | jq .iid)
+    BODY="{
+      \"id\": ${CI_PROJECT_ID},
+      \"iid\": ${MR_ID},
+      \"should_remove_source_branch\": true,
+      \"merge_when_pipeline_succeeds\": true.
+      \"target_branch\": \"test-MR\"
+      }";
+    RES=$(curl -s -X PUT "https://renkulab.io/gitlab/api/v4/projects/${CI_PROJECT_ID}/merge_requests/merge" \
+      --header "PRIVATE-TOKEN: ${CI_PUSH_TOKEN}" \
+      --header "Content-Type: application/json" \
+      --data "${BODY}")
+    echo $RES
+
+

Dockerfile

@@ -27,7 +27,7 @@ RUN conda env update -q -f /tmp/environment.yml && \
     jupyter labextension install --no-build @jupyter-widgets/jupyterlab-manager && \
     jupyter lab build && \
     conda clean -y --all && \
-    conda env export -n "root" 
+    conda env export -n "root"
 
 # install our utilities package
 
@@ -36,3 +36,7 @@ RUN ln -s /code/src
 RUN pip install -e src/covid-19/covid_19_utils
 
 RUN pipx install --force --pip-args="--pre" "renku==0.10.4.dev13"
+
+RUN wget -O ~/.local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 && \
+    chmod u+x ~/.local/bin/jq
+