.gitlab-ci.yml

@@ -5,13 +5,55 @@ variables:
 
 stages:
   - build
+  - test
 
 image_build:
   stage: build
   image: docker:stable
+  except:
+    - merge_requests 
   before_script:
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN http://$CI_REGISTRY
   script: |
     CI_COMMIT_SHA_7=$(echo $CI_COMMIT_SHA | cut -c1-7)
     docker build --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 .
-    docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7
+    docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7
\ No newline at end of file
+
+merge_request_build:
+  stage: build
+  image: docker:stable
+  only:
+    - merge_requests
+  before_script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN http://$CI_REGISTRY
+  script: |
+    docker build --tag ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-MR .
+    docker push ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-MR
+
+merge_request_test:
+  stage: test
+  image: 
+    name: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-MR
+    entrypoint: ['']
+  only:
+    - merge_requests
+  except:
+    # shouldnt run on forks automatically
+    - $PWD != /builds/gitlab/aaron.spring/s2s-ai-challenge-template/
+  before_script:
+    # install
+    - pip install pytest nbmake
+    # clone the project
+    - cd /tmp
+    - renku clone ${CI_REPOSITORY_URL}
+    - cd ${CI_PROJECT_NAME}
+    - echo "Source branch = ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}, Target branch = ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}"
+    # checkout to the source branch
+    - git checkout ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}
+    # pull lfs
+    - renku storage pull data/forecast-like-observations_2020_biweekly_terciled.nc
+    - renku storage pull submissions/ML_prediction_2020.nc
+    - renku storage pull data/ecmwf_recalibrated_benchmark_2020_biweekly_terciled.nc
+  script:
+    # testing notebook
+    - pytest --nbmake notebooks/RPSS_verification.ipynb

.renku/metadata.yml

@@ -2,7 +2,7 @@
 '@type':
 - http://schema.org/Project
 - http://www.w3.org/ns/prov#Location
-http://schema.org/agent: 0.14.2
+http://schema.org/agent: 0.16.0
 http://schema.org/creator:
   '@id': mailto:aaron.spring@mpimet.mpg.de
   '@type':

.renku/refs/datasets/s2s-ai-challenge

+../../datasets/a31a4922-5cbd-4d8e-bee3-435b1fd85282/metadata.yml
\ No newline at end of file

AUTHORS.md

+<!-- Please fill in your team name, your members names and email ids -->
+
+# Team `your GROUPNAME` members
+
+- Aaron Spring, aaron.spring@mpimet.mpg.de <!-- delete -->

CHANGELOG.md

 # CHANGELOG
 
+### 2021-10-22: `v0.4.1` *release*
+
+- After last minute community feedback (thank you [Sergi Bech Sala](https://renkulab.io/gitlab/sbechsal7) and [Llorenç Lledó](https://renkulab.io/gitlab/llledo)), we decided to modify the way we average the RPS for RPSS. Following [Weigel et al. 2007](https://doi.org/10.1175/MWR3280.1) [#50](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/issues/50), [!23](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/merge_requests/23) implements `RPSS=1-<RPS_ML>/<RPS_clim>`, whereas before we had `RPSS=<1-RPS_ML/RPS_clim>`, where angle brackets `<>` denote the average of the scores over a given number of forecast–observation pairs. Furthermore, we now penalize `NaNs` where numerical values where expected by `RPS=2`.
+- We restart the [`s2saichallengescorer`](https://renkulab.io/gitlab/tasko.olevski/s2s-ai-competition-scoring-image/-/tree/master), which automatically fetches the new scores of previous submissions, so your old submissions are re-evaluated. If you want to evaluate your score yourself, please use the updated [`skill_by_year`](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/scripts.py#L200).
+
+
+### 2021-07-27: `v0.4` *release*
+
+---
+**NOTE**
+
+After community feedback, we changed the ground truth observations files. Please update these files into your training pipeline if you used them before:
+- [hindcast-like-observations_2000-2019_biweekly_tercile-edges.nc](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/data/hindcast-like-observations_2000-2019_biweekly_tercile-edges.nc)
+- [hindcast-like-observations_2000-2019_biweekly_deterministic.zarr](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/tree/master/data/hindcast-like-observations_2000-2019_biweekly_deterministic.zarr)
+- [hindcast-like-observations_2000-2019_biweekly_terciled.zarr](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/tree/master/data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr)
+
+---
+
+
+- add `eccodes` to [`environment.yml`](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/environment.yml) to resolve `cfgrib` `ImportError` (https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/issues/26, !15, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+- Create categories by `groupby('week')`. (https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/issues/29, !18 [Aaron Spring](https://renkulab.io/gitlab/aaron.spring) with help from [Sergi Bech Sala](https://renkulab.io/gitlab/sbechsal7) and [Llorenç Lledó](https://renkulab.io/gitlab/llledo))
+- Regrid observations `conservative`ly with [xesmf](https://pangeo-xesmf.readthedocs.io/) (https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/issues/32, !18, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring) with help from [Arlan Dirkson](https://renkulab.io/gitlab/arlan.dirkson))
+- Observations now have data at `longitude=0` (https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/issues/35, !18, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring) with help from [Sergi Bech Sala](https://renkulab.io/gitlab/sbechsal7))
+- Missing `tp` observations after leap days added (https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/issues/31, !18, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring) with help from [David Landry](https://renkulab.io/gitlab/david.landry))
+- `conservative`ly regridded and `groupby('week')` observations are now also used by `s2saichallengescorer` (https://renkulab.io/gitlab/tasko.olevski/s2s-ai-competition-scoring-image/-/merge_requests/4, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+
+
+### 2021-06-19: `v0.3.1` *release*
+
+- Fix `sortby('forecast_time')` for renku datasets (as before). `renku dataset s2s-ai-challenge` tag equals repo tag (!15, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+
+
+### 2021-06-19: `v0.3.0` *release*
+
+- Recreate biweekly renku datasets with `float32` single precision and newly uploaded data to [`climetlab_s2s_ai_challenge`](https://github.com/ecmwf-lab/climetlab-s2s-ai-challenge/releases/tag/0.7.0). Before `lead_time` in `tp` had a one day shift. These new renku datasets will be used by the [`s2saichallengescorer`](https://renkulab.io/gitlab/tasko.olevski/s2s-ai-competition-scoring-image/-/merge_requests/3). (#3, #5, !14, https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/issues/22, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring)). Averaged recalibrated ECMWF RPSS skill value to beat at least: `RPSS = -0.043`, see [`RPSS_verification.ipynb`](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/RPSS_verification.ipynb). You should also beat `climatology`, i.e. `RPSS = 0`. 
+- Order of processing gridded `RPSS` to final score: (#7, !9, [s2s-ai-competition-scoring-image!2](https://renkulab.io/gitlab/tasko.olevski/s2s-ai-competition-scoring-image/-/merge_requests/2), [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+    1. `RPSS`
+    2. penalize where `NaN` submitted but value expected #7 
+    3. `clip(-10,1)`: prevent too negative values
+    4. mean over `forecast_time`
+    5. spatially weighted mean [90N-60S]
+    6. mean over `lead_time` and `data_vars`
+- Dont forget to `git add current_notebook.ipynb` also to ensure that consistent training pipeline and submission file are tagged, added to notebooks. (!9, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+- Rerun [`ML_train_and_predict.ipynb`](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/ML_train_and_predict.ipynb) (!9, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+- Fix typo in safeguards in [ML_forecast_template.ipynb](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/ML_forecast_template.ipynb): "We did NOT use `test` explicitly in training or implicitly in incrementally adjusting parameters." (!8, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+- Add notebooks showcasing accessing output of different models from different sources: (!2, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+    - S2S-Project models:
+        - from from European Weather Cloud:
+            - [`climetlab-s2s-ai-challenge`](https://github.com/ecmwf-lab/climetlab-s2s-ai-challenge/) [recommended], see [`climetlab-s2s-ai-challenge` notebooks](https://github.com/ecmwf-lab/climetlab-s2s-ai-challenge/tree/main/notebooks)
+            - `curl` & `wget`, see [wget_curl.ipynb](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/data_access/wget_curl.ipynb)
+            - `intake`, see [intake.ipynb](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/data_access/intake.ipynb)
+        - `IRIDL` including overview, see [IRIDL.ipynb](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/data_access/IRIDL.ipynb)
+        - `intake` catalogs for `IRIDL` [`SubX`](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/data_access/SubX_catalog.yml) and [`S2S`](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/data_access/S2S_catalog.yml) see [IRIDL.ipynb](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/data_access/IRIDL.ipynb) (#9 !10, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+    - SubX-Project models: `IRIDL` including overview, see [IRIDL.ipynb](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/data_access/IRIDL.ipynb)
+    - How to access password-protected S2S-Project output from `IRIDL` with `xarray`? see [IRIDL.ipynb](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/data_access/IRIDL.ipynb)
+    - Access NOAA CPC observations via opendap from `IRIDL`, see [IRIDL.ipynb](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/data_access/IRIDL.ipynb) (#9 !10, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+- fix `netcdf4` version to `1.5.4` for `opendap` to work lazily with `xarray` (!2, !7, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+
+
+### 2021-05-31: `v0.2` *release*
+
+After this `v0.2` release, this `CHANGELOG.md` will describe all changes made in this template repository.
+
+- update `README` how to join competition, please `git pull` if you forked before
+- find status of your submission in `s2s-ai-competition-scoring-image` https://renkulab.io/gitlab/tasko.olevski/s2s-ai-competition-scoring-image/-/blob/master/README.md 
+- calculate `RPSS` with respect to climatology (not ECMWF anymore) ([Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+    - update `RPSS_verification.ipynb`
+    - update `scorer`: https://renkulab.io/gitlab/tasko.olevski/s2s-ai-competition-scoring-image ([Tasko Olevski](https://renkulab.io/gitlab/tasko.olevski), [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+    - Averaged ECMWF RPSS skill value to beat at least: `-0.0070`
+
+
+### 2021-05-26: `v0.1` *pre-release*
+
+- update `README` how to join competition (!4, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+- git lfs track zarr: `git lfs track "**/*.zarr/**"` (!4, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+- add notebooks: (!4, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+    - create renku datasets: `renku_datasets_biweekly.ipynb`
+    - RPSS verification: `RPSS_verification.ipynb`
+    - ML train and predict based on weatherbench: `ML_train_and_predict.ipynb`
+    - mean bias reduction: `mean_bias_reduction.ipynb`
+    - template for training and predictions: `ML_forecast_template.ipynb`
+- add renku dataset `s2s-ai-challenge` with files: (!4, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+    - `hindcast-like-observations_2000-2019_biweekly_deterministic.zarr`
+    - `forecast-like-observations_2020_biweekly_deterministic.zarr`
+    - `hindcast-like-observations_2000-2019_biweekly_tercile-edges.nc`
+    - `hindcast-like-observations_2000-2019_biweekly_terciled.zarr`
+    - `forecast-like-observations_2020_biweekly_terciled.nc`
+    - `ecmwf_forecast-input_2020_biweekly_deterministic.zarr`
+    - `ecmwf_hindcast-input_2000-2019_biweekly_deterministic.zarr`
+    - `ecmwf_recalibrated_benchmark_2020_biweekly_terciled.nc`
+- add reproducibility section below in training (!4, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+- how to deal with this dry mask? provide as renku dataset? now implicitly masked in categorized observations `obs_p`
+- justify if training takes more than a week (!4, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+- show RPS for all years. ~~ToDo: take RPSS~~ (!4, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+
+
+
 ### 2021-05-10
 
 - `git lfs track “submissions/*.nc”`, so submission netcdf files are not carried in `git` but `git lfs`

Dockerfile

@@ -27,14 +27,24 @@ RUN conda env update -q -f /tmp/environment.yml && \
 # RENKU_VERSION determines the version of the renku CLI
 # that will be used in this image. To find the latest version,
 # visit https://pypi.org/project/renku/#history.
-ARG RENKU_VERSION=0.14.2
+ARG RENKU_VERSION=0.16.2
 
 ########################################################
 # Do not edit this section and do not add anything below
 
+# Install renku from pypi or from github if it's a dev version
 RUN if [ -n "$RENKU_VERSION" ] ; then \
-    pipx uninstall renku && \
-    pipx install --force renku==${RENKU_VERSION} \
-    ; fi
+        source .renku/venv/bin/activate ; \
+        currentversion=$(renku --version) ; \
+        if [ "$RENKU_VERSION" != "$currentversion" ] ; then \
+            pip uninstall renku ; \
+            gitversion=$(echo "$RENKU_VERSION" | sed -n "s/^[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+\(\.dev[[:digit:]]\+\)*\(+g\([a-f0-9]\+\)\)*\(+dirty\)*$/\3/p") ; \
+            if [ -n "$gitversion" ] ; then \
+                pip install --force "git+https://github.com/SwissDataScienceCenter/renku-python.git@$gitversion" ;\
+            else \
+                pip install --force renku==${RENKU_VERSION} pyshacl==0.17.0.post1 ;\
+            fi \
+        fi \
+    fi
 
-########################################################
+########################################################
\ No newline at end of file

README.md

@@ -5,23 +5,31 @@ the `s2s-ai-challenge`.
 
 You were likely referred here from the [public website](https://s2s-ai-challenge.github.io/).
 
-The competition starts in June, so examples are still work in progress
-and joining the competition not possible until then, but you can already look around.
-If you fork this project before June, please rebase or fork again in June.
+The submission period for the `s2s-ai-challenge` ended. Please find the steps to join the competition for documentation purposes below.
+However, the `s2saichallengescorer` still remains active. So you can still verify your 2020 predictions in the future and they will show up in the [RPSS leaderboard](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-leaderboard), but do not qualify for prizes.
 
-Find an overview of [repositories and websites](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/wikis/Flow-of-information:-Where-do-I-find-what%3F)
+---
+
+If you have already forked this project, please fork again or pull recent changes.
+Major changes will be also announced on the [challenge website](https://s2s-ai-challenge.github.io/#announcements).
+This template repository will have release tags to track changes.
+
+Find an overview of [repositories and websites](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/wikis/Flow-of-information:-Where-do-I-find-what%3F).
 
 ## Introduction
 
-This is a Renku project - basically a git repository with some
-bells and whistles. You'll find we have already created some
-useful things like `data` and `notebooks` directories and
+This is a Renku project. Renku is a platform for reproducible and collaborative data analysis.
+At its simplest a Renku project is a gitlab repository with added functionality.
+So you can use this project just as a gitlab repository if you wish. However, you may be surprised
+by what Renku has to offer and if you are curious the best place to start is the
+[Renku documentation](https://renku.readthedocs.io/en/latest/).
+You'll find we have already created some useful things like `data` and `notebooks` directories and
 a `Dockerfile`.
 
 ## Join the challenge
 
 ### 1. The simplest way to join the S2S AI Challenge is forking this renku project.
-Ensure you do not fork the gitlab repository, but the renku project.
+Ensure you fork the renku project and the underlying gitlab repository through the renkulab.io page.
 
 Fork this template renku project from https://renkulab.io/projects/aaron.spring/s2s-ai-challenge-template/settings.
 
@@ -29,34 +37,51 @@ Fork this template renku project from https://renkulab.io/projects/aaron.spring/
 
 Name your fork `s2s-ai-challenge-$TEAMNAME`.
 
+
+When cloning this repository and you do not want to immediately download the `git lfs`-backed [renku datasets](https://renku.readthedocs.io/projects/renku-python/en/v0.4.0/cli.html#module-renku.cli.dataset), please use:
+```bash
+GIT_LFS_SKIP_SMUDGE=1 renku/git clone https://renkulab.io/projects/$YOURNAME/s2s-ai-challenge-$TEAMNAME.git
+```
+
+To be able to pull future changes from the [template](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template) into your repository, add an `upstream`:
+
+```bash
+# in your fork locally
+git remote add upstream https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template.git
+git pull upstream master
+```
+
 ### 2. Fill our [registration form](https://docs.google.com/forms/d/1KEnATjaLOtV-o4N8PLinPXYnpba7egKsCCH_efriCb4).
 
+Registrations are not required before October 31st 2021, but highly [appreciated for the flow of information](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/issues/4).
+
 ### 3. Make the project private
 
-Now check out the gitlab repository by clicking on "View in gitlab".
+Now navigate to the gitlab page by clicking on "View in gitlab" in the upper right corner.
 Under "Settings" - "General" - "Visibility" you can set your project private.
 
 <img src="docs/screenshots/gitlab_visibility.png" width="300">
 
 Now other people cannot steal your idea/code.
 
-Now please modify the README in your fork with team details and a
+Please modify the `README` in your fork with your team's details and a
 description of your method.
 
-### 4. Add the `scorer` user to your repo with Reporter permissions
-The scorer is not yet ready, but will follow this [verification notebook](https://renkulab.io/gitlab/aaron.spring/s2s-ai-competition-bootstrap/-/blob/master/notebooks/verification_RPSS.ipynb).
 
-### 5. Add a gitlab variable with key `COMPETITION` and name `S2S-AI`
-In the gitlab repository, under "Settings" -> "CI/CD" -> "Variables", add the
-`COMPETITION` key with value `S2S-AI`, so the `scorer` bot knows where to search
-for submissions.
 
-todo: probably not needed instead ensure that project/repo has a certain `tag`
+### 4. Add the `s2saichallengescorer` user to your repo with Reporter permissions
+The scorer follows the code shown in the [verification notebook](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/verification_RPSS.ipynb). The scorer's username on gitlab is `s2saichallengescorer`. You should add it to your project with `Reporter` permissions. Under "Members" - "Invite Members" - "GitLab member or Email address", add `s2saichallengescorer`. The scorer will only ever clone your repository and evaluate your submission. It will never make any changes to your code.
 
-<img src="docs/screenshots/gitlab_variables.png" width="300">
-<img src="docs/screenshots/gitlab_add_variable.png" width="300">
+### 5. Add the `s2s-ai-challenge` topic to your repository
+To add the project topic navigate to `Settings` -> `General` and then fill in the word `s2s-ai-challenge` in the 
+`Topics` field near the top of the page. If you have multiple topics you can separate them by commas.
 
-## Contribute
+This allows your repository to be recognized as a participant of the competition. Without
+this project topic or if you have not added the scorer as a member of your project 
+the automated scoring bot will not evaluate any of your submissions and none of your code
+or results will be considered for the competition.
+
+## Make Predictions
 
 ### 6. Start jupyter on renku or locally
 The simplest way to contribute is right from the Renku platform - 
@@ -85,29 +110,50 @@ Get corresponding observations/ground truth:
 
 ### 8. Let the Machine Learning model perform subseasonal 2020 predictions
 and save them as `netcdf` files.
-The submissions have to placed in the `submissions` folder with filename `submission_your_choice.nc`,
-see [example](https://renkulab.io/gitlab/aaron.spring/s2s-ai-competition-bootstrap/-/blob/master/submissions/submission_rb_prediction_2020.nc).
+The submissions have to placed in the `submissions` folder with filename `ML_prediction_2020.nc`,
+see [example](https://renkulab.io/gitlab/aaron.spring/s2s-ai-competition-bootstrap/-/blob/master/submissions/ML_prediction_2020.nc).
 
 ### 9. `git commit` training pipeline and netcdf submission
-For later verification of the organizers, reproducibility and scoring of submissions,
-the training notebook/pipeline and submission file ML_prediction.nc with `git lfs`.
-After commiting, `git tag submission-method_name-number`
+For later verification by the organizers, reproducibility and scoring of submissions,
+commit all code, input and output data. For the data files please use `git lfs`. 
+If you are unfamiliar with `git lfs` a short introduction can be found 
+[here](https://www.atlassian.com/git/tutorials/git-lfs).
+This is very important because the organizers need to review and reliably reproduce 
+your results. If your results cannot be reliably reproduced then you cannot win
+the competition - even if your submitted results had the highest score. 
+
+After committing, tag your submission and push your commit and tag. The automated scorer will
+evaulate any tag (regardless of which branch it is on) that starts with the word `submission`
+followed by any other combination of characters. In other words, any tags that satisfy the
+regex `^submission.*` will be evaluated by the scorer. In addition, the scorer will only look for the
+results in a file named `ML_prediction_2020.nc` located in the `submissions` folder
+at the root of each competitor's repository.
+
+Here is an example of a set of commands that would commit the results and add the scorer tag.
 ```bash
-git lfs track "*.nc" # once, already done in template
-git add submissions/ML_prediction_2020.nc
-git commit -m "commit submission for my_method"
-git tag "submission-my_method-0.0.1" # if this is to be checked by scorer
+# run your training and create file ../submissions/ML_prediction_2020.nc
+git lfs track "*.nc"  # this will ensure that all *nc files are using lfs and needs to be done only once, already done in https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template
+git add submissions/ML_prediction_2020.nc  # submission file to be fetched by s2saichallengescorer
+git add notebooks/current_notebook.ipynb  # training and prediction notebook
+git commit -m "commit submission for my_method_name" # whatever message you want
+git tag "submission-my_method_name-0.0.1"  # add this tag if this is to be evaluated by the s2saichallengescorer
 git push --tags
 ```
 
-### 10. RPSS scoring by `scorer` bot
-The `scorer` will fetch your tagged submissions, score them with RPSS against recalibrated ECMWF real-time forecasts.
-Your score will be added to the private [leaderboard](https://s2s-ai-challenge.github.io/#leaderboard), which will be made public in early November 2021.
+Please note that only submitted/tagged commits will be considered for the competition.
+If you have code that produces better results after the competition ends and it has
+not been tagged or is tagged after the competition closed then this will not be considered.
+
+### 10. RPSS scoring by `s2saichallengescorer` bot
+The `s2saichallengescorer` will fetch your tagged submissions, score them with RPSS against recalibrated ECMWF real-time forecasts.
+Your score will be added to the private leaderboard, which will be made public in early November 2021.
 
-The `scorer` is not active for the competition yet.
+The `s2saichallengescorer` is not active for the competition yet.
 
 ## More information
-in the [wiki](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/wikis/Home)
+
+- in the [`s2s-ai-challenge` wiki](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/wikis/Home)
+- all different resources for this [competition](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/wikis/Flow-of-information:-Where-do-I-find-what%3F)
 
 ## Changing interactive environment dependencies