Compare revisions

Aaron Spring · Aaron Spring · Aaron Spring · Aaron Spring · Aaron Spring · Aaron Spring
--- a/.gitattributes
+++ b/.gitattributes
@@ -437,3 +437,91 @@ data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.0.8.0.0 f
 data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.0.7.0.0 filter=lfs diff=lfs merge=lfs -text
 data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.0.3.0.0 filter=lfs diff=lfs merge=lfs -text
 data/ecmwf_recalibrated_benchmark_2020_biweekly_terciled.nc filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.0.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.0.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.0.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.0.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.0.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.1.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.1.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.1.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.1.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.0.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.0.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.0.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.1.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.0.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.0.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.1.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.0.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.1.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.0.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.1.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.1.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.1.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.0.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.1.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.1.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.1.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.1.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.1.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.0.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.1.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.0.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.0.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.0.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.1.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.0.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.1.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.0.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.1.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.0.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.1.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.1.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.1.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.1.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.1.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.1.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.1.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.1.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.1.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.1.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.1.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.1.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.1.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.0.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.1.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.1.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.0.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.1.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.0.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.1.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.0.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.0.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.0.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.0.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.0.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/0.1.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.0.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.0.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.1.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.1.0.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.0.2.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.1.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.0.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.1.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.1.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.1.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.1.2.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.0.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.1.3.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/1.0.0.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.0.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/t2m/2.0.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.0.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.0.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/1.1.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.1.1.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.0.1.0.0 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/0.0.3.0.1 filter=lfs diff=lfs merge=lfs -text
+data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr/tp/2.1.2.0.1 filter=lfs diff=lfs merge=lfs -text
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,13 +5,55 @@ variables:

 stages:
  - build
+  - test

 image_build:
  stage: build
  image: docker:stable
+  except:
+    - merge_requests 
  before_script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN http://$CI_REGISTRY
  script: |
    CI_COMMIT_SHA_7=$(echo $CI_COMMIT_SHA | cut -c1-7)
    docker build --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7 .
-    docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7
\ No newline at end of file
+    docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA_7
+
+merge_request_build:
+  stage: build
+  image: docker:stable
+  only:
+    - merge_requests
+  before_script:
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN http://$CI_REGISTRY
+  script: |
+    docker build --tag ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-MR .
+    docker push ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-MR
+
+merge_request_test:
+  stage: test
+  image: 
+    name: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}-MR
+    entrypoint: ['']
+  only:
+    - merge_requests
+  except:
+    # shouldnt run on forks automatically
+    - $PWD != /builds/gitlab/aaron.spring/s2s-ai-challenge-template/
+  before_script:
+    # install
+    - pip install pytest nbmake
+    # clone the project
+    - cd /tmp
+    - renku clone ${CI_REPOSITORY_URL}
+    - cd ${CI_PROJECT_NAME}
+    - echo "Source branch = ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}, Target branch = ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}"
+    # checkout to the source branch
+    - git checkout ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}
+    # pull lfs
+    - renku storage pull data/forecast-like-observations_2020_biweekly_terciled.nc
+    - renku storage pull submissions/ML_prediction_2020.nc
+    - renku storage pull data/ecmwf_recalibrated_benchmark_2020_biweekly_terciled.nc
+  script:
+    # testing notebook
+    - pytest --nbmake notebooks/RPSS_verification.ipynb
--- a/.renku/datasets/a31a4922-5cbd-4d8e-bee3-435b1fd85282/metadata.yml
+++ b/.renku/datasets/a31a4922-5cbd-4d8e-bee3-435b1fd85282/metadata.yml
--- a/.renku/metadata.yml
+++ b/.renku/metadata.yml
@@ -2,7 +2,7 @@
 '@type':
 - http://schema.org/Project
 - http://www.w3.org/ns/prov#Location
-http://schema.org/agent: 0.15.1
+http://schema.org/agent: 0.16.0
 http://schema.org/creator:
  '@id': mailto:aaron.spring@mpimet.mpg.de
  '@type':

--- a/AUTHORS.md
+++ b/AUTHORS.md
+<!-- Please fill in your team name, your members names and email ids -->
+
+# Team `your GROUPNAME` members
+
+- Aaron Spring, aaron.spring@mpimet.mpg.de <!-- delete -->
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
 # CHANGELOG

-### unreleased
+### 2021-10-22: `v0.4.1` *release*

- Fix typo in safeguards in [ML_forecast_template.ipynb](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/ML_forecast_template.ipynb): "We did NOT use `test` explicitly in training or implicitly in incrementally adjusting parameters."" (!8, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+- After last minute community feedback (thank you [Sergi Bech Sala](https://renkulab.io/gitlab/sbechsal7) and [Llorenç Lledó](https://renkulab.io/gitlab/llledo)), we decided to modify the way we average the RPS for RPSS. Following [Weigel et al. 2007](https://doi.org/10.1175/MWR3280.1) [#50](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/issues/50), [!23](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/merge_requests/23) implements `RPSS=1-<RPS_ML>/<RPS_clim>`, whereas before we had `RPSS=<1-RPS_ML/RPS_clim>`, where angle brackets `<>` denote the average of the scores over a given number of forecast–observation pairs. Furthermore, we now penalize `NaNs` where numerical values where expected by `RPS=2`.
+- We restart the [`s2saichallengescorer`](https://renkulab.io/gitlab/tasko.olevski/s2s-ai-competition-scoring-image/-/tree/master), which automatically fetches the new scores of previous submissions, so your old submissions are re-evaluated. If you want to evaluate your score yourself, please use the updated [`skill_by_year`](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/scripts.py#L200).
+
+
+### 2021-07-27: `v0.4` *release*
+
+---
+**NOTE**
+
+After community feedback, we changed the ground truth observations files. Please update these files into your training pipeline if you used them before:
+- [hindcast-like-observations_2000-2019_biweekly_tercile-edges.nc](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/data/hindcast-like-observations_2000-2019_biweekly_tercile-edges.nc)
+- [hindcast-like-observations_2000-2019_biweekly_deterministic.zarr](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/tree/master/data/hindcast-like-observations_2000-2019_biweekly_deterministic.zarr)
+- [hindcast-like-observations_2000-2019_biweekly_terciled.zarr](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/tree/master/data/hindcast-like-observations_2000-2019_biweekly_terciled.zarr)
+
+---
+
+
+- add `eccodes` to [`environment.yml`](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/environment.yml) to resolve `cfgrib` `ImportError` (https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/issues/26, !15, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+- Create categories by `groupby('week')`. (https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/issues/29, !18 [Aaron Spring](https://renkulab.io/gitlab/aaron.spring) with help from [Sergi Bech Sala](https://renkulab.io/gitlab/sbechsal7) and [Llorenç Lledó](https://renkulab.io/gitlab/llledo))
+- Regrid observations `conservative`ly with [xesmf](https://pangeo-xesmf.readthedocs.io/) (https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/issues/32, !18, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring) with help from [Arlan Dirkson](https://renkulab.io/gitlab/arlan.dirkson))
+- Observations now have data at `longitude=0` (https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/issues/35, !18, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring) with help from [Sergi Bech Sala](https://renkulab.io/gitlab/sbechsal7))
+- Missing `tp` observations after leap days added (https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/issues/31, !18, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring) with help from [David Landry](https://renkulab.io/gitlab/david.landry))
+- `conservative`ly regridded and `groupby('week')` observations are now also used by `s2saichallengescorer` (https://renkulab.io/gitlab/tasko.olevski/s2s-ai-competition-scoring-image/-/merge_requests/4, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+
+
+### 2021-06-19: `v0.3.1` *release*
+
+- Fix `sortby('forecast_time')` for renku datasets (as before). `renku dataset s2s-ai-challenge` tag equals repo tag (!15, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+
+
+### 2021-06-19: `v0.3.0` *release*
+
+- Recreate biweekly renku datasets with `float32` single precision and newly uploaded data to [`climetlab_s2s_ai_challenge`](https://github.com/ecmwf-lab/climetlab-s2s-ai-challenge/releases/tag/0.7.0). Before `lead_time` in `tp` had a one day shift. These new renku datasets will be used by the [`s2saichallengescorer`](https://renkulab.io/gitlab/tasko.olevski/s2s-ai-competition-scoring-image/-/merge_requests/3). (#3, #5, !14, https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/issues/22, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring)). Averaged recalibrated ECMWF RPSS skill value to beat at least: `RPSS = -0.043`, see [`RPSS_verification.ipynb`](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/RPSS_verification.ipynb). You should also beat `climatology`, i.e. `RPSS = 0`. 
+- Order of processing gridded `RPSS` to final score: (#7, !9, [s2s-ai-competition-scoring-image!2](https://renkulab.io/gitlab/tasko.olevski/s2s-ai-competition-scoring-image/-/merge_requests/2), [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+    1. `RPSS`
+    2. penalize where `NaN` submitted but value expected #7 
+    3. `clip(-10,1)`: prevent too negative values
+    4. mean over `forecast_time`
+    5. spatially weighted mean [90N-60S]
+    6. mean over `lead_time` and `data_vars`
+- Dont forget to `git add current_notebook.ipynb` also to ensure that consistent training pipeline and submission file are tagged, added to notebooks. (!9, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+- Rerun [`ML_train_and_predict.ipynb`](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/ML_train_and_predict.ipynb) (!9, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
+- Fix typo in safeguards in [ML_forecast_template.ipynb](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/ML_forecast_template.ipynb): "We did NOT use `test` explicitly in training or implicitly in incrementally adjusting parameters." (!8, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
 - Add notebooks showcasing accessing output of different models from different sources: (!2, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
    - S2S-Project models:
        - from from European Weather Cloud:
@@ -10,21 +52,23 @@
            - `curl` & `wget`, see [wget_curl.ipynb](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/data_access/wget_curl.ipynb)
            - `intake`, see [intake.ipynb](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/data_access/intake.ipynb)
        - `IRIDL` including overview, see [IRIDL.ipynb](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/data_access/IRIDL.ipynb)
+        - `intake` catalogs for `IRIDL` [`SubX`](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/data_access/SubX_catalog.yml) and [`S2S`](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/data_access/S2S_catalog.yml) see [IRIDL.ipynb](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/data_access/IRIDL.ipynb) (#9 !10, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
    - SubX-Project models: `IRIDL` including overview, see [IRIDL.ipynb](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/data_access/IRIDL.ipynb)
-    - How to access password-protected S2S-Project output from IRIDL with xarray? see [IRIDL.ipynb](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/data_access/IRIDL.ipynb)
+    - How to access password-protected S2S-Project output from `IRIDL` with `xarray`? see [IRIDL.ipynb](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/data_access/IRIDL.ipynb)
+    - Access NOAA CPC observations via opendap from `IRIDL`, see [IRIDL.ipynb](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/data_access/IRIDL.ipynb) (#9 !10, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
 - fix `netcdf4` version to `1.5.4` for `opendap` to work lazily with `xarray` (!2, !7, [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))


 ### 2021-05-31: `v0.2` *release*

-After this `v0.2` release, this CHANGELOG.md will describe all changes made in this template repository.
+After this `v0.2` release, this `CHANGELOG.md` will describe all changes made in this template repository.

 - update `README` how to join competition, please `git pull` if you forked before
 - find status of your submission in `s2s-ai-competition-scoring-image` https://renkulab.io/gitlab/tasko.olevski/s2s-ai-competition-scoring-image/-/blob/master/README.md 
 - calculate `RPSS` with respect to climatology (not ECMWF anymore) ([Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
    - update `RPSS_verification.ipynb`
    - update `scorer`: https://renkulab.io/gitlab/tasko.olevski/s2s-ai-competition-scoring-image ([Tasko Olevski](https://renkulab.io/gitlab/tasko.olevski), [Aaron Spring](https://renkulab.io/gitlab/aaron.spring))
-    - Averaged ECMWF RPSS skill value to beat at least: -0.0070
+    - Averaged ECMWF RPSS skill value to beat at least: `-0.0070`


 ### 2021-05-26: `v0.1` *pre-release*

--- a/Dockerfile
+++ b/Dockerfile
@@ -27,14 +27,24 @@ RUN conda env update -q -f /tmp/environment.yml && \
 # RENKU_VERSION determines the version of the renku CLI
 # that will be used in this image. To find the latest version,
 # visit https://pypi.org/project/renku/#history.
-ARG RENKU_VERSION=0.15.1
+ARG RENKU_VERSION=0.16.2

 ########################################################
 # Do not edit this section and do not add anything below

+# Install renku from pypi or from github if it's a dev version
 RUN if [ -n "$RENKU_VERSION" ] ; then \
-    pipx uninstall renku && \
-    pipx install --force renku==${RENKU_VERSION} \
-    ; fi
+        source .renku/venv/bin/activate ; \
+        currentversion=$(renku --version) ; \
+        if [ "$RENKU_VERSION" != "$currentversion" ] ; then \
+            pip uninstall renku ; \
+            gitversion=$(echo "$RENKU_VERSION" | sed -n "s/^[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+\(\.dev[[:digit:]]\+\)*\(+g\([a-f0-9]\+\)\)*\(+dirty\)*$/\3/p") ; \
+            if [ -n "$gitversion" ] ; then \
+                pip install --force "git+https://github.com/SwissDataScienceCenter/renku-python.git@$gitversion" ;\
+            else \
+                pip install --force renku==${RENKU_VERSION} pyshacl==0.17.0.post1 ;\
+            fi \
+        fi \
+    fi

-########################################################
\ No newline at end of file
+########################################################
--- a/README.md
+++ b/README.md
@@ -5,9 +5,14 @@ the `s2s-ai-challenge`.

 You were likely referred here from the [public website](https://s2s-ai-challenge.github.io/).

-If you have already forked this project before June 1st 2021, please fork again or pull recent changes.
+The submission period for the `s2s-ai-challenge` ended. Please find the steps to join the competition for documentation purposes below.
+However, the `s2saichallengescorer` still remains active. So you can still verify your 2020 predictions in the future and they will show up in the [RPSS leaderboard](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-leaderboard), but do not qualify for prizes.
+
+---
+
+If you have already forked this project, please fork again or pull recent changes.
 Major changes will be also announced on the [challenge website](https://s2s-ai-challenge.github.io/#announcements).
-This template repository will have release tags for orientation.
+This template repository will have release tags to track changes.

 Find an overview of [repositories and websites](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge/-/wikis/Flow-of-information:-Where-do-I-find-what%3F).

@@ -64,7 +69,7 @@ description of your method.



-### 4. Add the `scorer` user to your repo with Reporter permissions
+### 4. Add the `s2saichallengescorer` user to your repo with Reporter permissions
 The scorer follows the code shown in the [verification notebook](https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template/-/blob/master/notebooks/verification_RPSS.ipynb). The scorer's username on gitlab is `s2saichallengescorer`. You should add it to your project with `Reporter` permissions. Under "Members" - "Invite Members" - "GitLab member or Email address", add `s2saichallengescorer`. The scorer will only ever clone your repository and evaluate your submission. It will never make any changes to your code.

 ### 5. Add the `s2s-ai-challenge` topic to your repository
@@ -127,10 +132,11 @@ at the root of each competitor's repository.
 Here is an example of a set of commands that would commit the results and add the scorer tag.
 ```bash
 # run your training and create file ../submissions/ML_prediction_2020.nc
-git lfs track "*.nc" # this will ensure that all *nc files are using lfs and needs to be done only once
-git add submissions/ML_prediction_2020.nc
+git lfs track "*.nc"  # this will ensure that all *nc files are using lfs and needs to be done only once, already done in https://renkulab.io/gitlab/aaron.spring/s2s-ai-challenge-template
+git add submissions/ML_prediction_2020.nc  # submission file to be fetched by s2saichallengescorer
+git add notebooks/current_notebook.ipynb  # training and prediction notebook
 git commit -m "commit submission for my_method_name" # whatever message you want
-git tag "submission-my_method_name-0.0.1" # add this tag if this is to be evaluated by the scorer
+git tag "submission-my_method_name-0.0.1"  # add this tag if this is to be evaluated by the s2saichallengescorer
 git push --tags
 ```

@@ -138,11 +144,11 @@ Please note that only submitted/tagged commits will be considered for the compet
 If you have code that produces better results after the competition ends and it has
 not been tagged or is tagged after the competition closed then this will not be considered.

-### 10. RPSS scoring by `scorer` bot
-The `scorer` will fetch your tagged submissions, score them with RPSS against recalibrated ECMWF real-time forecasts.
+### 10. RPSS scoring by `s2saichallengescorer` bot
+The `s2saichallengescorer` will fetch your tagged submissions, score them with RPSS against recalibrated ECMWF real-time forecasts.
 Your score will be added to the private leaderboard, which will be made public in early November 2021.

-The `scorer` is not active for the competition yet.
+The `s2saichallengescorer` is not active for the competition yet.

 ## More information


--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/.zattrs
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/.zattrs
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/.zmetadata
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/.zmetadata
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/forecast_time/.zattrs
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/forecast_time/.zattrs
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/lead_time/.zattrs
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/lead_time/.zattrs
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/.zarray
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/.zarray
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/.zattrs
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/.zattrs
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.0.0.0
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.0.0.0
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.1.0.0
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.1.0.0
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.2.0.0
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.2.0.0
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.3.0.0
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.3.0.0
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.4.0.0
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/t2m/0.0.4.0.0
--- a/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/tp/.zarray
+++ b/data/ecmwf_forecast-input_2020_biweekly_deterministic.zarr/tp/.zarray
No results found