From 96f193e0ba61438cdf8033f16d8d4d03a2c6aad7 Mon Sep 17 00:00:00 2001 From: SimonKamuk <43374850+SimonKamuk@users.noreply.github.com> Date: Mon, 10 Jun 2024 11:24:58 +0200 Subject: [PATCH 1/7] Run ci/cd tests on push to main (#55) This is a minor change for ci/cd to also run on pushes to main (which then includes the push created when a branch is merged). Also changed ci/cd badges to only look at the main branch. --- .github/workflows/pre-commit.yml | 4 +--- .github/workflows/run_tests.yml | 4 +--- CHANGELOG.md | 4 ++-- README.md | 4 ++-- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index dadac50d..ad2b1a9c 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -1,10 +1,8 @@ name: Linting on: - # trigger on pushes to any branch, but not main + # trigger on pushes to any branch push: - branches-ignore: - - main # and also on PRs to main pull_request: branches: diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 71bff3d3..4c677908 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -1,10 +1,8 @@ name: Unit Tests on: - # trigger on pushes to any branch, but not main + # trigger on pushes to any branch push: - branches-ignore: - - main # and also on PRs to main pull_request: branches: diff --git a/CHANGELOG.md b/CHANGELOG.md index f4680c37..d109dcb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased](https://github.com/joeloskarsson/neural-lam/compare/v0.1.0...HEAD) ### Added -- Added tests for loading dataset, creating graph, and training model based on reduced MEPS dataset stored on AWS S3, along with automatic running of tests on push/PR to GitHub. Added caching of test data tp speed up running tests. - [/#38](https://github.com/mllam/neural-lam/pull/38) +- Added tests for loading dataset, creating graph, and training model based on reduced MEPS dataset stored on AWS S3, along with automatic running of tests on push/PR to GitHub, including push to main branch. Added caching of test data to speed up running tests. + [\#38](https://github.com/mllam/neural-lam/pull/38) [\#55](https://github.com/mllam/neural-lam/pull/55) @SimonKamuk - Replaced `constants.py` with `data_config.yaml` for data configuration management diff --git a/README.md b/README.md index 1bdc6602..26d844f7 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -![Linting](https://github.com/mllam/neural-lam/actions/workflows/pre-commit.yml/badge.svg) -![Automatic tests](https://github.com/mllam/neural-lam/actions/workflows/run_tests.yml/badge.svg) +![Linting](https://github.com/mllam/neural-lam/actions/workflows/pre-commit.yml/badge.svg?branch=main) +![Automatic tests](https://github.com/mllam/neural-lam/actions/workflows/run_tests.yml/badge.svg?branch=main)

From 066efe0a77f1ed741d9eb2f303ee920927d79ad2 Mon Sep 17 00:00:00 2001 From: sadamov <45732287+sadamov@users.noreply.github.com> Date: Thu, 13 Jun 2024 08:47:56 +0200 Subject: [PATCH 2/7] Add entry for PR #22 (previously forgotten) (#56) Changelog updated with missing entry for #22 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d109dcb9..34a8e0e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed + Optional multi-core/GPU support for statistics calculation in `create_parameter_weights.py` + [\#22](https://github.com/mllam/neural-lam/pull/22) + @sadamov + - Robust restoration of optimizer and scheduler using `ckpt_path` [\#17](https://github.com/mllam/neural-lam/pull/17) @sadamov From a4e9a3b6fb3909bfa015511a05562c3d8140f670 Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Thu, 11 Jul 2024 21:29:14 +0200 Subject: [PATCH 3/7] Add draft github pull-request template (#53) Add a pull-request template for use on github when adding pull-requests to the common repo --------- Co-authored-by: Joel Oskarsson Co-authored-by: sadamov <45732287+sadamov@users.noreply.github.com> --- .github/pull_request_template.md | 52 ++++++++++++++++++++++++++++++++ CHANGELOG.md | 3 ++ 2 files changed, 55 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..b4bf15ea --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,52 @@ +## Describe your changes + +< Summary of the changes.> + +< Please also include relevant motivation and context. > + +< List any dependencies that are required for this change. > + +## Issue Link + +< Link to the relevant issue or task. > (e.g. `closes #00` or `solves #00`) + +## Type of change + +- [ ] 🐛 Bug fix (non-breaking change that fixes an issue) +- [ ] ✨ New feature (non-breaking change that adds functionality) +- [ ] 💥 Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] 📖 Documentation (Addition or improvements to documentation) + +## Checklist before requesting a review + +- [ ] My branch is up-to-date with the target branch - if not update your fork with the changes from the target branch (use `pull` with `--rebase` option if possible). +- [ ] I have performed a self-review of my code +- [ ] For any new/modified functions/classes I have added docstrings that clearly describe its purpose, expected inputs and returned values +- [ ] I have placed in-line comments to clarify the intent of any hard-to-understand passages of my code +- [ ] I have updated the [README](README.MD) to cover introduced code changes +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] I have given the PR a name that clearly describes the change, written in imperative form ([context](https://www.gitkraken.com/learn/git/best-practices/git-commit-message#using-imperative-verb-form)). +- [ ] I have requested a reviewer and an assignee (assignee is responsible for merging) + +## Checklist for reviewers + +Each PR comes with its own improvements and flaws. The reviewer should check the following: +- [ ] the code is readable +- [ ] the code is well tested +- [ ] the code is documented (including return types and parameters) +- [ ] the code is easy to maintain + +## Author checklist after completed review + +- [ ] I have added a line to the CHANGELOG describing this change, in a section + reflecting type of change (add section where missing): + - *added*: when you have added new functionality + - *changed*: when default behaviour of the code has been changed + - *fixes*: when your contribution fixes a bug + +## Checklist for assignee + +- [ ] PR is up to date with the base branch +- [ ] the tests pass +- [ ] author has added an entry to the changelog (and designated the change as *added*, *changed* or *fixed*) +- Once the PR is ready to be merged, squash commits and merge the PR. diff --git a/CHANGELOG.md b/CHANGELOG.md index 34a8e0e4..69140a11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [\#6](https://github.com/joeloskarsson/neural-lam/pull/6), [\#8](https://github.com/joeloskarsson/neural-lam/pull/8) @sadamov, @joeloskarsson +- added github pull-request template to ease contribution and review process + [\#53](https://github.com/mllam/neural-lam/pull/53), @leifdenby + ### Changed Optional multi-core/GPU support for statistics calculation in `create_parameter_weights.py` From 72965a9a654b79bf40c3828c3aef51b61e2c1a25 Mon Sep 17 00:00:00 2001 From: Joel Oskarsson Date: Thu, 8 Aug 2024 10:46:15 +0200 Subject: [PATCH 4/7] Cap numpy version to less than 2.0.0 (#68) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Describe your changes Nerual-lam is at the moment not completely compatible with the new release of numpy 2. This PR changes the numpy version in requirements.txt to < 2.0.0. ## Issue Link https://github.com/mllam/neural-lam/issues/67 ## Type of change - [x] 🐛 Bug fix (non-breaking change that fixes an issue) - [ ] ✨ New feature (non-breaking change that adds functionality) - [ ] 💥 Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] 📖 Documentation (Addition or improvements to documentation) ## Checklist before requesting a review - [x] My branch is up-to-date with the target branch - if not update your fork with the changes from the target branch (use `pull` with `--rebase` option if possible). - [x] I have performed a self-review of my code - [x] I have given the PR a name that clearly describes the change, written in imperative form ([context](https://www.gitkraken.com/learn/git/best-practices/git-commit-message#using-imperative-verb-form)). - [x] I have requested a reviewer and an assignee (assignee is responsible for merging) ## Checklist for reviewers Each PR comes with its own improvements and flaws. The reviewer should check the following: - [x] the code is readable - [x] the code is well tested - [x] the code is documented (including return types and parameters) - [x] the code is easy to maintain ## Author checklist after completed review - [x] I have added a line to the CHANGELOG describing this change, in a section reflecting type of change (add section where missing): - *added*: when you have added new functionality - *changed*: when default behaviour of the code has been changed - *fixes*: when your contribution fixes a bug ## Checklist for assignee - [x] PR is up to date with the base branch - [x] the tests pass - [x] author has added an entry to the changelog (and designated the change as *added*, *changed* or *fixed*) - Once the PR is ready to be merged, squash commits and merge the PR. --- CHANGELOG.md | 4 ++++ requirements.txt | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69140a11..dfb186f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [\#52](https://github.com/mllam/neural-lam/pull/52) @joeloskarsson +- Cap numpy version to < 2.0.0 + [\#68](https://github.com/mllam/neural-lam/pull/68) + @joeloskarsson + ## [v0.1.0](https://github.com/joeloskarsson/neural-lam/releases/tag/v0.1.0) First tagged release of `neural-lam`, matching Oskarsson et al 2023 publication diff --git a/requirements.txt b/requirements.txt index 9309eea4..6bcf304d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # for all -numpy>=1.24.2 +numpy>=1.24.2, <2.0.0 wandb>=0.13.10 matplotlib>=3.7.0 scipy>=1.10.0 From a54c45f82812207e3eb9da7b4e5baafb2e7b4441 Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Mon, 19 Aug 2024 15:44:39 +0200 Subject: [PATCH 5/7] Refactor codebase into a python package (#32) Make it possible to `neural-lam` as a package, thereby enabling the possibility to run from anywhere once the package has been installed. This means it is now possible (in theory) to train neural-lam on a `.npy`-file based dataset with the neural-lam package installed into a user's `site-packages` (i.e. in their virtualenv). The primary changes are: - move all `*.py` that are currently outside of neural_lam/ into that folder, but keep the files the same - change all examples of running the neural-lam "scripts", e.g. `python create_mesh.py` by `python -m neural_lam.create_mesh` in the README - change all absolute imports to package-relative imports, i.e. `from .import utils` rather than `from neural_lam import utils` - add tests that all the CLI entrypoints to neural_lam can be imported and add ci/cd action to run these tests --------- Co-authored-by: SimonKamuk <43374850+SimonKamuk@users.noreply.github.com> Co-authored-by: joeloskarsson Co-authored-by: Leif Denby --- .github/workflows/run_tests.yml | 2 +- .pre-commit-config.yaml | 1 + CHANGELOG.md | 7 ++ README.md | 94 +++++++++---------- neural_lam/__init__.py | 10 ++ .../create_grid_features.py | 4 +- create_mesh.py => neural_lam/create_mesh.py | 4 +- .../create_parameter_weights.py | 5 +- neural_lam/interaction_net.py | 4 +- neural_lam/models/__init__.py | 6 ++ neural_lam/models/ar_model.py | 4 +- neural_lam/models/base_graph_model.py | 8 +- neural_lam/models/base_hi_graph_model.py | 8 +- neural_lam/models/graph_lam.py | 8 +- neural_lam/models/hi_lam.py | 6 +- neural_lam/models/hi_lam_parallel.py | 6 +- train_model.py => neural_lam/train_model.py | 9 +- neural_lam/vis.py | 4 +- neural_lam/weather_dataset.py | 4 +- pyproject.toml | 7 ++ tests/__init__.py | 0 tests/test_cli.py | 18 ++++ tests/test_mllam_dataset.py | 20 ++-- 23 files changed, 141 insertions(+), 98 deletions(-) create mode 100644 neural_lam/__init__.py rename create_grid_features.py => neural_lam/create_grid_features.py (97%) rename create_mesh.py => neural_lam/create_mesh.py (99%) rename create_parameter_weights.py => neural_lam/create_parameter_weights.py (99%) create mode 100644 neural_lam/models/__init__.py rename train_model.py => neural_lam/train_model.py (97%) delete mode 100644 tests/__init__.py create mode 100644 tests/test_cli.py diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 4c677908..810f2b2c 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -35,7 +35,7 @@ jobs: ${{ runner.os }}-meps-reduced-example-data-v0.1.0 - name: Test with pytest run: | - pytest -v -s + python -m pytest -v -s tests/ - name: Save cache data uses: actions/cache/save@v4 with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 815a92e1..dfbf8b60 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,3 +35,4 @@ repos: hooks: - id: flake8 description: Check Python code for correctness, consistency and adherence to best practices + additional_dependencies: [Flake8-pyproject] diff --git a/CHANGELOG.md b/CHANGELOG.md index dfb186f7..c183888e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,6 +92,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [\#68](https://github.com/mllam/neural-lam/pull/68) @joeloskarsson +- turn `neural-lam` into a python package by moving all `*.py`-files into the + `neural_lam/` source directory and updating imports accordingly. This means + all cli functions are now invoke through the package name, e.g. `python -m + neural_lam.train_model` instead of `python train_model.py` (and can be done + anywhere once the package has been installed). + [\#32](https://github.com/mllam/neural-lam/pull/32), @leifdenby + ## [v0.1.0](https://github.com/joeloskarsson/neural-lam/releases/tag/v0.1.0) First tagged release of `neural-lam`, matching Oskarsson et al 2023 publication diff --git a/README.md b/README.md index 26d844f7..ce8daf69 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Still, some restrictions are inevitable: ## A note on the limited area setting Currently we are using these models on a limited area covering the Nordic region, the so called MEPS area (see [paper](https://arxiv.org/abs/2309.17370)). There are still some parts of the code that is quite specific for the MEPS area use case. -This is in particular true for the mesh graph creation (`create_mesh.py`) and some of the constants set in a `data_config.yaml` file (path specified in `train_model.py --data_config` ). +This is in particular true for the mesh graph creation (`python -m neural_lam.create_mesh`) and some of the constants set in a `data_config.yaml` file (path specified in `python -m neural_lam.train_model --data_config ` ). If there is interest to use Neural-LAM for other areas it is not a substantial undertaking to refactor the code to be fully area-agnostic. We would be happy to support such enhancements. See the issues https://github.com/joeloskarsson/neural-lam/issues/2, https://github.com/joeloskarsson/neural-lam/issues/3 and https://github.com/joeloskarsson/neural-lam/issues/4 for some initial ideas on how this could be done. @@ -62,16 +62,10 @@ Follow the steps below to create the necessary python environment. 1. Install GEOS for your system. For example with `sudo apt-get install libgeos-dev`. This is necessary for the Cartopy requirement. 2. Use python 3.9. 3. Install version 2.0.1 of PyTorch. Follow instructions on the [PyTorch webpage](https://pytorch.org/get-started/previous-versions/) for how to set this up with GPU support on your system. -4. Install required packages specified in `requirements.txt`. -5. Install PyTorch Geometric version 2.2.0. This can be done by running +4. Install `neural-lam` with pip: ``` -TORCH="2.0.1" -CUDA="cu117" - -pip install pyg-lib==0.2.0 torch-scatter==2.1.1 torch-sparse==0.6.17 torch-cluster==1.6.1\ - torch-geometric==2.3.1 -f https://pytorch-geometric.com/whl/torch-${TORCH}+${CUDA}.html +pip install -e . ``` -You will have to adjust the `CUDA` variable to match the CUDA version on your system or to run on CPU. See the [installation webpage](https://pytorch-geometric.readthedocs.io/en/latest/install/installation.html) for more information. ## Data Datasets should be stored in a directory called `data`. @@ -80,39 +74,39 @@ See the [repository format section](#format-of-data-directory) for details on th The full MEPS dataset can be shared with other researchers on request, contact us for this. A tiny subset of the data (named `meps_example`) is available in `example_data.zip`, which can be downloaded from [here](https://liuonline-my.sharepoint.com/:f:/g/personal/joeos82_liu_se/EuiUuiGzFIFHruPWpfxfUmYBSjhqMUjNExlJi9W6ULMZ1w?e=97pnGX). Download the file and unzip in the neural-lam directory. -All graphs used in the paper are also available for download at the same link (but can as easily be re-generated using `create_mesh.py`). -Note that this is far too little data to train any useful models, but all scripts can be ran with it. +All graphs used in the paper are also available for download at the same link (but can as easily be re-generated using `python -m neural_lam.create_mesh`). +Note that this is far too little data to train any useful models, but all pre-processing and training steps can be run with it. It should thus be useful to make sure that your python environment is set up correctly and that all the code can be ran without any issues. ## Pre-processing -An overview of how the different scripts and files depend on each other is given in this figure: +An overview of how the different pre-processing steps, training and files depend on each other is given in this figure:

-In order to start training models at least three pre-processing scripts have to be ran: +In order to start training models at least three pre-processing steps have to be run: -* `create_mesh.py` -* `create_grid_features.py` -* `create_parameter_weights.py` +* `python -m neural_lam.create_mesh` +* `python -m neural_lam.create_grid_features` +* `python -m neural_lam.create_parameter_weights` ### Create graph -Run `create_mesh.py` with suitable options to generate the graph you want to use (see `python create_mesh.py --help` for a list of options). +Run `python -m neural_lam.create_mesh` with suitable options to generate the graph you want to use (see `python neural_lam.create_mesh --help` for a list of options). The graphs used for the different models in the [paper](https://arxiv.org/abs/2309.17370) can be created as: -* **GC-LAM**: `python create_mesh.py --graph multiscale` -* **Hi-LAM**: `python create_mesh.py --graph hierarchical --hierarchical 1` (also works for Hi-LAM-Parallel) -* **L1-LAM**: `python create_mesh.py --graph 1level --levels 1` +* **GC-LAM**: `python -m neural_lam.create_mesh --graph multiscale` +* **Hi-LAM**: `python -m neural_lam.create_mesh --graph hierarchical --hierarchical 1` (also works for Hi-LAM-Parallel) +* **L1-LAM**: `python -m neural_lam.create_mesh --graph 1level --levels 1` The graph-related files are stored in a directory called `graphs`. ### Create remaining static features -To create the remaining static files run the scripts `create_grid_features.py` and `create_parameter_weights.py`. +To create the remaining static files run `python -m neural_lam.create_grid_features` and `python -m neural_lam.create_parameter_weights`. ## Weights & Biases Integration The project is fully integrated with [Weights & Biases](https://www.wandb.ai/) (W&B) for logging and visualization, but can just as easily be used without it. When W&B is used, training configuration, training/test statistics and plots are sent to the W&B servers and made available in an interactive web interface. If W&B is turned off, logging instead saves everything locally to a directory like `wandb/dryrun...`. -The W&B project name is set to `neural-lam`, but this can be changed in the flags of `train_model.py` (using argsparse). +The W&B project name is set to `neural-lam`, but this can be changed in the flags of `python -m neural_lam.train_model` (using argsparse). See the [W&B documentation](https://docs.wandb.ai/) for details. If you would like to login and use W&B, run: @@ -125,8 +119,8 @@ wandb off ``` ## Train Models -Models can be trained using `train_model.py`. -Run `python train_model.py --help` for a full list of training options. +Models can be trained using `python -m neural_lam.train_model`. +Run `python neural_lam.train_model --help` for a full list of training options. A few of the key ones are outlined below: * `--dataset`: Which data to train on @@ -145,12 +139,12 @@ This model class is used both for the L1-LAM and GC-LAM models from the [paper]( To train 1L-LAM use ``` -python train_model.py --model graph_lam --graph 1level ... +python -m neural_lam.train_model --model graph_lam --graph 1level ... ``` To train GC-LAM use ``` -python train_model.py --model graph_lam --graph multiscale ... +python -m neural_lam.train_model --model graph_lam --graph multiscale ... ``` ### Hi-LAM @@ -158,7 +152,7 @@ A version of Graph-LAM that uses a hierarchical mesh graph and performs sequenti To train Hi-LAM use ``` -python train_model.py --model hi_lam --graph hierarchical ... +python -m neural_lam.train_model --model hi_lam --graph hierarchical ... ``` ### Hi-LAM-Parallel @@ -167,13 +161,13 @@ Not included in the paper as initial experiments showed worse results than Hi-LA To train Hi-LAM-Parallel use ``` -python train_model.py --model hi_lam_parallel --graph hierarchical ... +python -m neural_lam.train_model --model hi_lam_parallel --graph hierarchical ... ``` Checkpoint files for our models trained on the MEPS data are available upon request. ## Evaluate Models -Evaluation is also done using `train_model.py`, but using the `--eval` option. +Evaluation is also done using `python -m neural_lam.train_model`, but using the `--eval` option. Use `--eval val` to evaluate the model on the validation set and `--eval test` to evaluate on test data. Most of the training options are also relevant for evaluation (not `ar_steps`, evaluation always unrolls full forecasts). Some options specifically important for evaluation are: @@ -216,13 +210,13 @@ data │ ├── nwp_xy.npy - Coordinates of grid nodes (part of dataset) │ ├── surface_geopotential.npy - Geopotential at surface of grid nodes (part of dataset) │ ├── border_mask.npy - Mask with True for grid nodes that are part of border (part of dataset) -│ ├── grid_features.pt - Static features of grid nodes (create_grid_features.py) -│ ├── parameter_mean.pt - Means of state parameters (create_parameter_weights.py) -│ ├── parameter_std.pt - Std.-dev. of state parameters (create_parameter_weights.py) -│ ├── diff_mean.pt - Means of one-step differences (create_parameter_weights.py) -│ ├── diff_std.pt - Std.-dev. of one-step differences (create_parameter_weights.py) -│ ├── flux_stats.pt - Mean and std.-dev. of solar flux forcing (create_parameter_weights.py) -│ └── parameter_weights.npy - Loss weights for different state parameters (create_parameter_weights.py) +│ ├── grid_features.pt - Static features of grid nodes (neural_lam.create_grid_features) +│ ├── parameter_mean.pt - Means of state parameters (neural_lam.create_parameter_weights) +│ ├── parameter_std.pt - Std.-dev. of state parameters (neural_lam.create_parameter_weights) +│ ├── diff_mean.pt - Means of one-step differences (neural_lam.create_parameter_weights) +│ ├── diff_std.pt - Std.-dev. of one-step differences (neural_lam.create_parameter_weights) +│ ├── flux_stats.pt - Mean and std.-dev. of solar flux forcing (neural_lam.create_parameter_weights) +│ └── parameter_weights.npy - Loss weights for different state parameters (neural_lam.create_parameter_weights) ├── dataset2 ├── ... └── datasetN @@ -234,13 +228,13 @@ The structure is shown with examples below: ``` graphs ├── graph1 - Directory with a graph definition -│ ├── m2m_edge_index.pt - Edges in mesh graph (create_mesh.py) -│ ├── g2m_edge_index.pt - Edges from grid to mesh (create_mesh.py) -│ ├── m2g_edge_index.pt - Edges from mesh to grid (create_mesh.py) -│ ├── m2m_features.pt - Static features of mesh edges (create_mesh.py) -│ ├── g2m_features.pt - Static features of grid to mesh edges (create_mesh.py) -│ ├── m2g_features.pt - Static features of mesh to grid edges (create_mesh.py) -│ └── mesh_features.pt - Static features of mesh nodes (create_mesh.py) +│ ├── m2m_edge_index.pt - Edges in mesh graph (neural_lam.create_mesh) +│ ├── g2m_edge_index.pt - Edges from grid to mesh (neural_lam.create_mesh) +│ ├── m2g_edge_index.pt - Edges from mesh to grid (neural_lam.create_mesh) +│ ├── m2m_features.pt - Static features of mesh edges (neural_lam.create_mesh) +│ ├── g2m_features.pt - Static features of grid to mesh edges (neural_lam.create_mesh) +│ ├── m2g_features.pt - Static features of mesh to grid edges (neural_lam.create_mesh) +│ └── mesh_features.pt - Static features of mesh nodes (neural_lam.create_mesh) ├── graph2 ├── ... └── graphN @@ -250,9 +244,9 @@ graphs To keep track of levels in the mesh graph, a list format is used for the files with mesh graph information. In particular, the files ``` -│ ├── m2m_edge_index.pt - Edges in mesh graph (create_mesh.py) -│ ├── m2m_features.pt - Static features of mesh edges (create_mesh.py) -│ ├── mesh_features.pt - Static features of mesh nodes (create_mesh.py) +│ ├── m2m_edge_index.pt - Edges in mesh graph (neural_lam.create_mesh) +│ ├── m2m_features.pt - Static features of mesh edges (neural_lam.create_mesh) +│ ├── mesh_features.pt - Static features of mesh nodes (neural_lam.create_mesh) ``` all contain lists of length `L`, for a hierarchical mesh graph with `L` layers. For non-hierarchical graphs `L == 1` and these are all just singly-entry lists. @@ -263,10 +257,10 @@ In addition, hierarchical mesh graphs (`L > 1`) feature a few additional files w ``` ├── graph1 │ ├── ... -│ ├── mesh_down_edge_index.pt - Downward edges in mesh graph (create_mesh.py) -│ ├── mesh_up_edge_index.pt - Upward edges in mesh graph (create_mesh.py) -│ ├── mesh_down_features.pt - Static features of downward mesh edges (create_mesh.py) -│ ├── mesh_up_features.pt - Static features of upward mesh edges (create_mesh.py) +│ ├── mesh_down_edge_index.pt - Downward edges in mesh graph (neural_lam.create_mesh) +│ ├── mesh_up_edge_index.pt - Upward edges in mesh graph (neural_lam.create_mesh) +│ ├── mesh_down_features.pt - Static features of downward mesh edges (neural_lam.create_mesh) +│ ├── mesh_up_features.pt - Static features of upward mesh edges (neural_lam.create_mesh) │ ├── ... ``` These files have the same list format as the ones above, but each list has length `L-1` (as these edges describe connections between levels). diff --git a/neural_lam/__init__.py b/neural_lam/__init__.py new file mode 100644 index 00000000..dd565a26 --- /dev/null +++ b/neural_lam/__init__.py @@ -0,0 +1,10 @@ +# First-party +import neural_lam.config +import neural_lam.interaction_net +import neural_lam.metrics +import neural_lam.models +import neural_lam.utils +import neural_lam.vis + +# Local +from .weather_dataset import WeatherDataset diff --git a/create_grid_features.py b/neural_lam/create_grid_features.py similarity index 97% rename from create_grid_features.py rename to neural_lam/create_grid_features.py index 4f058e17..adabd9dc 100644 --- a/create_grid_features.py +++ b/neural_lam/create_grid_features.py @@ -6,8 +6,8 @@ import numpy as np import torch -# First-party -from neural_lam import config +# Local +from . import config def main(): diff --git a/create_mesh.py b/neural_lam/create_mesh.py similarity index 99% rename from create_mesh.py rename to neural_lam/create_mesh.py index 41557a97..40f7ba0e 100644 --- a/create_mesh.py +++ b/neural_lam/create_mesh.py @@ -12,8 +12,8 @@ import torch_geometric as pyg from torch_geometric.utils.convert import from_networkx -# First-party -from neural_lam import config +# Local +from . import config def plot_graph(graph, title=None): diff --git a/create_parameter_weights.py b/neural_lam/create_parameter_weights.py similarity index 99% rename from create_parameter_weights.py rename to neural_lam/create_parameter_weights.py index c85cd5a3..74058d38 100644 --- a/create_parameter_weights.py +++ b/neural_lam/create_parameter_weights.py @@ -10,9 +10,8 @@ from torch.utils.data.distributed import DistributedSampler from tqdm import tqdm -# First-party -from neural_lam import config -from neural_lam.weather_dataset import WeatherDataset +# Local +from . import WeatherDataset, config class PaddedWeatherDataset(torch.utils.data.Dataset): diff --git a/neural_lam/interaction_net.py b/neural_lam/interaction_net.py index 663f27e4..2f45b03f 100644 --- a/neural_lam/interaction_net.py +++ b/neural_lam/interaction_net.py @@ -3,8 +3,8 @@ import torch_geometric as pyg from torch import nn -# First-party -from neural_lam import utils +# Local +from . import utils class InteractionNet(pyg.nn.MessagePassing): diff --git a/neural_lam/models/__init__.py b/neural_lam/models/__init__.py new file mode 100644 index 00000000..f65387ab --- /dev/null +++ b/neural_lam/models/__init__.py @@ -0,0 +1,6 @@ +# Local +from .base_graph_model import BaseGraphModel +from .base_hi_graph_model import BaseHiGraphModel +from .graph_lam import GraphLAM +from .hi_lam import HiLAM +from .hi_lam_parallel import HiLAMParallel diff --git a/neural_lam/models/ar_model.py b/neural_lam/models/ar_model.py index 6ced211f..e94de8c6 100644 --- a/neural_lam/models/ar_model.py +++ b/neural_lam/models/ar_model.py @@ -8,8 +8,8 @@ import torch import wandb -# First-party -from neural_lam import config, metrics, utils, vis +# Local +from .. import config, metrics, utils, vis class ARModel(pl.LightningModule): diff --git a/neural_lam/models/base_graph_model.py b/neural_lam/models/base_graph_model.py index 256d4adc..99629073 100644 --- a/neural_lam/models/base_graph_model.py +++ b/neural_lam/models/base_graph_model.py @@ -1,10 +1,10 @@ # Third-party import torch -# First-party -from neural_lam import utils -from neural_lam.interaction_net import InteractionNet -from neural_lam.models.ar_model import ARModel +# Local +from .. import utils +from ..interaction_net import InteractionNet +from .ar_model import ARModel class BaseGraphModel(ARModel): diff --git a/neural_lam/models/base_hi_graph_model.py b/neural_lam/models/base_hi_graph_model.py index 3fd30579..a2ebcc1b 100644 --- a/neural_lam/models/base_hi_graph_model.py +++ b/neural_lam/models/base_hi_graph_model.py @@ -1,10 +1,10 @@ # Third-party from torch import nn -# First-party -from neural_lam import utils -from neural_lam.interaction_net import InteractionNet -from neural_lam.models.base_graph_model import BaseGraphModel +# Local +from .. import utils +from ..interaction_net import InteractionNet +from .base_graph_model import BaseGraphModel class BaseHiGraphModel(BaseGraphModel): diff --git a/neural_lam/models/graph_lam.py b/neural_lam/models/graph_lam.py index f767fba0..d73f7ad8 100644 --- a/neural_lam/models/graph_lam.py +++ b/neural_lam/models/graph_lam.py @@ -1,10 +1,10 @@ # Third-party import torch_geometric as pyg -# First-party -from neural_lam import utils -from neural_lam.interaction_net import InteractionNet -from neural_lam.models.base_graph_model import BaseGraphModel +# Local +from .. import utils +from ..interaction_net import InteractionNet +from .base_graph_model import BaseGraphModel class GraphLAM(BaseGraphModel): diff --git a/neural_lam/models/hi_lam.py b/neural_lam/models/hi_lam.py index 4d7eb94c..4f3aec05 100644 --- a/neural_lam/models/hi_lam.py +++ b/neural_lam/models/hi_lam.py @@ -1,9 +1,9 @@ # Third-party from torch import nn -# First-party -from neural_lam.interaction_net import InteractionNet -from neural_lam.models.base_hi_graph_model import BaseHiGraphModel +# Local +from ..interaction_net import InteractionNet +from .base_hi_graph_model import BaseHiGraphModel class HiLAM(BaseHiGraphModel): diff --git a/neural_lam/models/hi_lam_parallel.py b/neural_lam/models/hi_lam_parallel.py index 740824e1..b40a9424 100644 --- a/neural_lam/models/hi_lam_parallel.py +++ b/neural_lam/models/hi_lam_parallel.py @@ -2,9 +2,9 @@ import torch import torch_geometric as pyg -# First-party -from neural_lam.interaction_net import InteractionNet -from neural_lam.models.base_hi_graph_model import BaseHiGraphModel +# Local +from ..interaction_net import InteractionNet +from .base_hi_graph_model import BaseHiGraphModel class HiLAMParallel(BaseHiGraphModel): diff --git a/train_model.py b/neural_lam/train_model.py similarity index 97% rename from train_model.py rename to neural_lam/train_model.py index 03863275..39f7aecd 100644 --- a/train_model.py +++ b/neural_lam/train_model.py @@ -9,12 +9,9 @@ import torch from lightning_fabric.utilities import seed -# First-party -from neural_lam import config, utils -from neural_lam.models.graph_lam import GraphLAM -from neural_lam.models.hi_lam import HiLAM -from neural_lam.models.hi_lam_parallel import HiLAMParallel -from neural_lam.weather_dataset import WeatherDataset +# Local +from . import WeatherDataset, config, utils +from .models import GraphLAM, HiLAM, HiLAMParallel MODELS = { "graph_lam": GraphLAM, diff --git a/neural_lam/vis.py b/neural_lam/vis.py index 8c9ca77c..2f22bef1 100644 --- a/neural_lam/vis.py +++ b/neural_lam/vis.py @@ -3,8 +3,8 @@ import matplotlib.pyplot as plt import numpy as np -# First-party -from neural_lam import utils +# Local +from . import utils @matplotlib.rc_context(utils.fractional_plot_bundle(1)) diff --git a/neural_lam/weather_dataset.py b/neural_lam/weather_dataset.py index 3288ed67..29977789 100644 --- a/neural_lam/weather_dataset.py +++ b/neural_lam/weather_dataset.py @@ -7,8 +7,8 @@ import numpy as np import torch -# First-party -from neural_lam import utils +# Local +from . import utils class WeatherDataset(torch.utils.data.Dataset): diff --git a/pyproject.toml b/pyproject.toml index b513a258..c482abc9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,10 @@ +[project] +name = "neural-lam" +version = "0.1.0" + +[tool.setuptools] +py-modules = ["neural_lam"] + [tool.black] line-length = 80 diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 00000000..e90daa04 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,18 @@ +# First-party +import neural_lam +import neural_lam.create_grid_features +import neural_lam.create_mesh +import neural_lam.create_parameter_weights +import neural_lam.train_model + + +def test_import(): + """ + This test just ensures that each cli entry-point can be imported for now, + eventually we should test their execution too + """ + assert neural_lam is not None + assert neural_lam.create_mesh is not None + assert neural_lam.create_grid_features is not None + assert neural_lam.create_parameter_weights is not None + assert neural_lam.train_model is not None diff --git a/tests/test_mllam_dataset.py b/tests/test_mllam_dataset.py index f91170c9..e12a57ae 100644 --- a/tests/test_mllam_dataset.py +++ b/tests/test_mllam_dataset.py @@ -1,15 +1,17 @@ # Standard library import os +from pathlib import Path # Third-party import pooch +import pytest # First-party -from create_mesh import main as create_mesh from neural_lam.config import Config +from neural_lam.create_mesh import main as create_mesh +from neural_lam.train_model import main as train_model from neural_lam.utils import load_static_data from neural_lam.weather_dataset import WeatherDataset -from train_model import main as train_model # Disable weights and biases to avoid unnecessary logging # and to avoid having to deal with authentication @@ -25,7 +27,8 @@ ) -def test_retrieve_data_ewc(): +@pytest.fixture(scope="module") +def meps_example_reduced_filepath(): # Download and unzip test data into data/meps_example_reduced pooch.retrieve( url=S3_FULL_PATH, @@ -34,16 +37,17 @@ def test_retrieve_data_ewc(): path="data", fname="meps_example_reduced.zip", ) + return Path("data/meps_example_reduced") -def test_load_reduced_meps_dataset(): +def test_load_reduced_meps_dataset(meps_example_reduced_filepath): # The data_config.yaml file is downloaded and extracted in # test_retrieve_data_ewc together with the dataset itself - data_config_file = "data/meps_example_reduced/data_config.yaml" - dataset_name = "meps_example_reduced" + data_config_file = meps_example_reduced_filepath / "data_config.yaml" + dataset_name = meps_example_reduced_filepath.name - dataset = WeatherDataset(dataset_name="meps_example_reduced") - config = Config.from_file(data_config_file) + dataset = WeatherDataset(dataset_name=dataset_name) + config = Config.from_file(str(data_config_file)) var_names = config.values["dataset"]["var_names"] var_units = config.values["dataset"]["var_units"] From 4969f92ad974f136089d15e7e2e2e9d73a43590d Mon Sep 17 00:00:00 2001 From: Leif Denby Date: Tue, 20 Aug 2024 14:32:09 +0200 Subject: [PATCH 6/7] Move deps to pyproject toml and setup ci/cd cpu+gpu install testing (#37) Move dependencies from `requirements.txt` to`pyproject.toml`, introduce pdm/pip tests for both CPU and GPU based testing to ensure this new way of defining dependencies works for both CPU and GPU installs (GPU tests running on AWS EC2) and remove cap on numpy version (to make it clear we support `numpy >= 2.0.0`. --- .cirun.yml | 16 +++++ .../workflows/ci-pdm-install-and-test-cpu.yml | 55 +++++++++++++++++ .../workflows/ci-pdm-install-and-test-gpu.yml | 60 +++++++++++++++++++ .../workflows/ci-pip-install-and-test-cpu.yml | 45 ++++++++++++++ .../workflows/ci-pip-install-and-test-gpu.yml | 50 ++++++++++++++++ .github/workflows/run_tests.yml | 43 ------------- .gitignore | 5 ++ CHANGELOG.md | 11 +++- README.md | 42 ++++++++++--- pyproject.toml | 37 ++++++++++++ requirements.txt | 17 ------ tests/test_imports.py | 8 +++ 12 files changed, 319 insertions(+), 70 deletions(-) create mode 100644 .cirun.yml create mode 100644 .github/workflows/ci-pdm-install-and-test-cpu.yml create mode 100644 .github/workflows/ci-pdm-install-and-test-gpu.yml create mode 100644 .github/workflows/ci-pip-install-and-test-cpu.yml create mode 100644 .github/workflows/ci-pip-install-and-test-gpu.yml delete mode 100644 .github/workflows/run_tests.yml delete mode 100644 requirements.txt create mode 100644 tests/test_imports.py diff --git a/.cirun.yml b/.cirun.yml new file mode 100644 index 00000000..21b03ab4 --- /dev/null +++ b/.cirun.yml @@ -0,0 +1,16 @@ +# setup for using github runners via https://cirun.io/ +runners: + - name: "aws-runner" + # Cloud Provider: AWS + cloud: "aws" + # https://aws.amazon.com/ec2/instance-types/g4/ + instance_type: "g4ad.xlarge" + # Deep Learning Base OSS Nvidia Driver GPU AMI (Ubuntu 22.04), Frankfurt region + machine_image: "ami-0ba41b554b28d24a4" + # use Frankfurt region + region: "eu-central-1" + preemptible: false + # Add this label in the "runs-on" param in .github/workflows/.yml + # So that this runner is created for running the workflow + labels: + - "cirun-aws-runner" diff --git a/.github/workflows/ci-pdm-install-and-test-cpu.yml b/.github/workflows/ci-pdm-install-and-test-cpu.yml new file mode 100644 index 00000000..c5da88cc --- /dev/null +++ b/.github/workflows/ci-pdm-install-and-test-cpu.yml @@ -0,0 +1,55 @@ +# cicd workflow for running tests with pytest +# needs to first install pdm, then install torch cpu manually and then install the package +# then run the tests + +name: test (pdm install, cpu) + +on: [push, pull_request] + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Install pdm + run: | + python -m pip install pdm + + - name: Create venv + run: | + pdm venv create --with-pip + pdm use --venv in-project + + - name: Install torch (CPU) + run: | + pdm run python -m pip install torch --index-url https://download.pytorch.org/whl/cpu + # check that the CPU version is installed + + - name: Install package (including dev dependencies) + run: | + pdm install --group :all + + - name: Print and check torch version + run: | + pdm run python -c "import torch; print(torch.__version__)" + pdm run python -c "import torch; assert torch.__version__.endswith('+cpu')" + + - name: Load cache data + uses: actions/cache/restore@v4 + with: + path: data + key: ${{ runner.os }}-meps-reduced-example-data-v0.1.0 + restore-keys: | + ${{ runner.os }}-meps-reduced-example-data-v0.1.0 + + - name: Run tests + run: | + pdm run pytest + + - name: Save cache data + uses: actions/cache/save@v4 + with: + path: data + key: ${{ runner.os }}-meps-reduced-example-data-v0.1.0 diff --git a/.github/workflows/ci-pdm-install-and-test-gpu.yml b/.github/workflows/ci-pdm-install-and-test-gpu.yml new file mode 100644 index 00000000..9ab4f379 --- /dev/null +++ b/.github/workflows/ci-pdm-install-and-test-gpu.yml @@ -0,0 +1,60 @@ +# cicd workflow for running tests with pytest +# needs to first install pdm, then install torch cpu manually and then install the package +# then run the tests + +name: test (pdm install, gpu) + +on: [push, pull_request] + +jobs: + tests: + runs-on: "cirun-aws-runner--${{ github.run_id }}" + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Install pdm + run: | + python -m pip install pdm + + - name: Create venv + run: | + pdm config venv.in_project False + pdm config venv.location /opt/dlami/nvme/venv + pdm venv create --with-pip + + - name: Install torch (GPU CUDA 12.1) + run: | + pdm run python -m pip install torch --index-url https://download.pytorch.org/whl/cu121 + + - name: Print and check torch version + run: | + pdm run python -c "import torch; print(torch.__version__)" + pdm run python -c "import torch; assert not torch.__version__.endswith('+cpu')" + + - name: Install package (including dev dependencies) + run: | + pdm install --group :all + + - name: Load cache data + uses: actions/cache/restore@v4 + with: + path: data + key: ${{ runner.os }}-meps-reduced-example-data-v0.1.0 + restore-keys: | + ${{ runner.os }}-meps-reduced-example-data-v0.1.0 + + - name: Run tests + run: | + pdm run pytest + + - name: Save cache data + uses: actions/cache/save@v4 + with: + path: data + key: ${{ runner.os }}-meps-reduced-example-data-v0.1.0 diff --git a/.github/workflows/ci-pip-install-and-test-cpu.yml b/.github/workflows/ci-pip-install-and-test-cpu.yml new file mode 100644 index 00000000..81e402c5 --- /dev/null +++ b/.github/workflows/ci-pip-install-and-test-cpu.yml @@ -0,0 +1,45 @@ +# cicd workflow for running tests with pytest +# needs to first install pdm, then install torch cpu manually and then install the package +# then run the tests + +name: test (pip install, cpu) + +on: [push, pull_request] + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Install torch (CPU) + run: | + python -m pip install torch --index-url https://download.pytorch.org/whl/cpu + + - name: Install package (including dev dependencies) + run: | + python -m pip install ".[dev]" + + - name: Print and check torch version + run: | + python -c "import torch; print(torch.__version__)" + python -c "import torch; assert torch.__version__.endswith('+cpu')" + + - name: Load cache data + uses: actions/cache/restore@v4 + with: + path: data + key: ${{ runner.os }}-meps-reduced-example-data-v0.1.0 + restore-keys: | + ${{ runner.os }}-meps-reduced-example-data-v0.1.0 + + - name: Run tests + run: | + python -m pytest + + - name: Save cache data + uses: actions/cache/save@v4 + with: + path: data + key: ${{ runner.os }}-meps-reduced-example-data-v0.1.0 diff --git a/.github/workflows/ci-pip-install-and-test-gpu.yml b/.github/workflows/ci-pip-install-and-test-gpu.yml new file mode 100644 index 00000000..ce68946a --- /dev/null +++ b/.github/workflows/ci-pip-install-and-test-gpu.yml @@ -0,0 +1,50 @@ +# cicd workflow for running tests with pytest +# needs to first install pdm, then install torch cpu manually and then install the package +# then run the tests + +name: test (pip install, gpu) + +on: [push, pull_request] + +jobs: + tests: + runs-on: "cirun-aws-runner--${{ github.run_id }}" + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Install torch (GPU CUDA 12.1) + run: | + python -m pip install torch --index-url https://download.pytorch.org/whl/cu121 + + - name: Install package (including dev dependencies) + run: | + python -m pip install ".[dev]" + + - name: Print and check torch version + run: | + python -c "import torch; print(torch.__version__)" + python -c "import torch; assert not torch.__version__.endswith('+cpu')" + + - name: Load cache data + uses: actions/cache/restore@v4 + with: + path: data + key: ${{ runner.os }}-meps-reduced-example-data-v0.1.0 + restore-keys: | + ${{ runner.os }}-meps-reduced-example-data-v0.1.0 + + - name: Run tests + run: | + python -m pytest + + - name: Save cache data + uses: actions/cache/save@v4 + with: + path: data + key: ${{ runner.os }}-meps-reduced-example-data-v0.1.0 diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml deleted file mode 100644 index 810f2b2c..00000000 --- a/.github/workflows/run_tests.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Unit Tests - -on: - # trigger on pushes to any branch - push: - # and also on PRs to main - pull_request: - branches: - - main - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - pip install torch-geometric>=2.5.2 - - name: Load cache data - uses: actions/cache/restore@v4 - with: - path: data - key: ${{ runner.os }}-meps-reduced-example-data-v0.1.0 - restore-keys: | - ${{ runner.os }}-meps-reduced-example-data-v0.1.0 - - name: Test with pytest - run: | - python -m pytest -v -s tests/ - - name: Save cache data - uses: actions/cache/save@v4 - with: - path: data - key: ${{ runner.os }}-meps-reduced-example-data-v0.1.0 diff --git a/.gitignore b/.gitignore index 65e9f6f8..022206f5 100644 --- a/.gitignore +++ b/.gitignore @@ -75,3 +75,8 @@ tags # Coc configuration directory .vim + +# pdm (https://pdm-project.org/en/stable/) +.pdm-python +# exclude pdm.lock file so that both cpu and gpu versions of torch will be accepted by pdm +pdm.lock diff --git a/CHANGELOG.md b/CHANGELOG.md index c183888e..f7c5cd63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - added github pull-request template to ease contribution and review process [\#53](https://github.com/mllam/neural-lam/pull/53), @leifdenby +- ci/cd setup for running both CPU and GPU-based testing both with pdm and pip based installs [\#37](https://github.com/mllam/neural-lam/pull/37), @khintz, @leifdenby + ### Changed Optional multi-core/GPU support for statistics calculation in `create_parameter_weights.py` @@ -88,10 +90,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [\#52](https://github.com/mllam/neural-lam/pull/52) @joeloskarsson -- Cap numpy version to < 2.0.0 +- Cap numpy version to < 2.0.0 (this cap was removed in #37, see below) [\#68](https://github.com/mllam/neural-lam/pull/68) @joeloskarsson +- Remove numpy < 2.0.0 version cap + [\#37](https://github.com/mllam/neural-lam/pull/37) + @leifdenby + - turn `neural-lam` into a python package by moving all `*.py`-files into the `neural_lam/` source directory and updating imports accordingly. This means all cli functions are now invoke through the package name, e.g. `python -m @@ -99,6 +105,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 anywhere once the package has been installed). [\#32](https://github.com/mllam/neural-lam/pull/32), @leifdenby +- move from `requirements.txt` to `pyproject.toml` for defining package dependencies. + [\#37](https://github.com/mllam/neural-lam/pull/37), @leifdenby + ## [v0.1.0](https://github.com/joeloskarsson/neural-lam/releases/tag/v0.1.0) First tagged release of `neural-lam`, matching Oskarsson et al 2023 publication diff --git a/README.md b/README.md index ce8daf69..7dc6c7ab 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ ![Linting](https://github.com/mllam/neural-lam/actions/workflows/pre-commit.yml/badge.svg?branch=main) -![Automatic tests](https://github.com/mllam/neural-lam/actions/workflows/run_tests.yml/badge.svg?branch=main) +[![test (pdm install, gpu)](https://github.com/mllam/neural-lam/actions/workflows/ci-pdm-install-and-test-gpu.yml/badge.svg)](https://github.com/mllam/neural-lam/actions/workflows/ci-pdm-install-and-test-gpu.yml) +[![test (pdm install, cpu)](https://github.com/mllam/neural-lam/actions/workflows/ci-pdm-install-and-test-cpu.yml/badge.svg)](https://github.com/mllam/neural-lam/actions/workflows/ci-pdm-install-and-test-cpu.yml)

@@ -57,15 +58,38 @@ See the issues https://github.com/joeloskarsson/neural-lam/issues/2, https://git Below follows instructions on how to use Neural-LAM to train and evaluate models. ## Installation -Follow the steps below to create the necessary python environment. -1. Install GEOS for your system. For example with `sudo apt-get install libgeos-dev`. This is necessary for the Cartopy requirement. -2. Use python 3.9. -3. Install version 2.0.1 of PyTorch. Follow instructions on the [PyTorch webpage](https://pytorch.org/get-started/previous-versions/) for how to set this up with GPU support on your system. -4. Install `neural-lam` with pip: -``` -pip install -e . -``` +When installing `neural-lam` you have a choice of either installing with +directly `pip` or using the `pdm` package manager. +We recommend using `pdm` as it makes it easy to add/remove packages while +keeping versions consistent (it automatically updates the `pyproject.toml` +file), makes it easy to handle virtual environments and includes the +development toolchain packages installation too. + +**regarding `torch` installation**: because `torch` creates different package +variants for different CUDA versions and cpu-only support you will need to install +`torch` separately if you don't want the most recent GPU variant that also +expects the most recent version of CUDA on your system. + +We cover all the installation options in our [github actions ci/cd +setup](.github/workflows/) which you can use as a reference. + +### Using `pdm` + +1. Clone this repository and navigate to the root directory. +2. Install `pdm` if you don't have it installed on your system (either with `pip install pdm` or [following the install instructions](https://pdm-project.org/latest/#installation)). +> If you are happy using the latest version of `torch` with GPU support (expecting the latest version of CUDA is installed on your system) you can skip to step 5. +3. Create a virtual environment for pdm to use with `pdm venv create --with-pip`. +4. Install a specific version of `torch` with `pdm run python -m pip install torch --index-url https://download.pytorch.org/whl/cpu` for a CPU-only version or `pdm run python -m pip install torch --index-url https://download.pytorch.org/whl/cu111` for CUDA 11.1 support (you can find the correct URL for the variant you want on [PyTorch webpage](https://pytorch.org/get-started/locally/)). +5. Install the dependencies with `pdm install` (by default this in include the). If you will be developing `neural-lam` we recommend to install the development dependencies with `pdm install --group dev`. By default `pdm` installs the `neural-lam` package in editable mode, so you can make changes to the code and see the effects immediately. + +### Using `pip` + +1. Clone this repository and navigate to the root directory. +> If you are happy using the latest version of `torch` with GPU support (expecting the latest version of CUDA is installed on your system) you can skip to step 3. +2. Install a specific version of `torch` with `python -m pip install torch --index-url https://download.pytorch.org/whl/cpu` for a CPU-only version or `python -m pip install torch --index-url https://download.pytorch.org/whl/cu111` for CUDA 11.1 support (you can find the correct URL for the variant you want on [PyTorch webpage](https://pytorch.org/get-started/locally/)). +3. Install the dependencies with `python -m pip install .`. If you will be developing `neural-lam` we recommend to install in editable mode and install the development dependencies with `python -m pip install -e ".[dev]"` so you can make changes to the code and see the effects immediately. + ## Data Datasets should be stored in a directory called `data`. diff --git a/pyproject.toml b/pyproject.toml index c482abc9..d66c0087 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,38 @@ [project] name = "neural-lam" version = "0.1.0" +description = "LAM-based data-driven forecasting" +authors = [ + {name = "Joel Oskarsson", email = "joel.oskarsson@liu.se"}, + {name = "Simon Adamov", email = "Simon.Adamov@meteoswiss.ch"}, + {name = "Leif Denby", email = "lcd@dmi.dk"}, +] + +# PEP 621 project metadata +# See https://www.python.org/dev/peps/pep-0621/ +dependencies = [ + "numpy>=1.24.2", + "wandb>=0.13.10", + "scipy>=1.10.0", + "pytorch-lightning>=2.0.3", + "shapely>=2.0.1", + "networkx>=3.0", + "Cartopy>=0.22.0", + "pyproj>=3.4.1", + "tueplots>=0.0.8", + "matplotlib>=3.7.0", + "plotly>=5.15.0", + "torch>=2.3.0", + "torch-geometric==2.3.1", +] +requires-python = ">=3.9" +[project.optional-dependencies] +dev = [ + "pre-commit>=3.8.0", + "pytest>=8.3.2", + "pooch>=1.8.2", +] [tool.setuptools] py-modules = ["neural_lam"] @@ -70,3 +101,9 @@ max-statements=100 # Allow for some more involved functions allow-any-import-level="neural_lam" [tool.pylint.SIMILARITIES] min-similarity-lines=10 + + +[tool.pdm] +[build-system] +requires = ["pdm-backend"] +build-backend = "pdm.backend" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 6bcf304d..00000000 --- a/requirements.txt +++ /dev/null @@ -1,17 +0,0 @@ -# for all -numpy>=1.24.2, <2.0.0 -wandb>=0.13.10 -matplotlib>=3.7.0 -scipy>=1.10.0 -pytorch-lightning>=2.0.3 -shapely>=2.0.1 -networkx>=3.0 -Cartopy>=0.22.0 -pyproj>=3.4.1 -tueplots>=0.0.8 -plotly>=5.15.0 - -# for dev -pre-commit>=2.15.0 -pytest>=8.1.1 -pooch>=1.8.1 diff --git a/tests/test_imports.py b/tests/test_imports.py new file mode 100644 index 00000000..e7bbd356 --- /dev/null +++ b/tests/test_imports.py @@ -0,0 +1,8 @@ +# First-party +import neural_lam +import neural_lam.vis + + +def test_import(): + assert neural_lam is not None + assert neural_lam.vis is not None From 68399f7e0a3964bc73ec798b8b3c08cd761a3eb1 Mon Sep 17 00:00:00 2001 From: Erik Larsson <86654747+ErikLarssonDev@users.noreply.github.com> Date: Thu, 5 Sep 2024 13:59:26 +0200 Subject: [PATCH 7/7] Update Argument Parser to use action="store_true" instead of 0/1 for boolean arguments. (#72) I have replaced the parser.add_argument where the standard has been to have an integer 0=False, 1=True with an action="store_true" instead. Example: OLD: parser.add_argument( "--restore_opt", type=int, default=0, help="If optimizer state should be restored with model " "(default: 0 (false))", ) NEW: parser.add_argument( "--restore_opt", action="store_true", help="If optimizer state should be restored with model " "(default: false)", ) This will save some time and characters when running the scripts from the command line as well as being easier to understand as the parsed variables are supposed to be booleans. --- CHANGELOG.md | 6 +++++- README.md | 2 +- neural_lam/create_mesh.py | 10 ++++------ neural_lam/create_parameter_weights.py | 5 ++--- neural_lam/train_model.py | 26 +++++++++++--------------- plot_graph.py | 5 ++--- tests/test_mllam_dataset.py | 2 +- 7 files changed, 26 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7c5cd63..7348f6a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,7 +35,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Optional multi-core/GPU support for statistics calculation in `create_parameter_weights.py` +- Argument Parser updated to use action="store_true" instead of 0/1 for boolean arguments. + (https://github.com/mllam/neural-lam/pull/72) + @ErikLarssonDev + +- Optional multi-core/GPU support for statistics calculation in `create_parameter_weights.py` [\#22](https://github.com/mllam/neural-lam/pull/22) @sadamov diff --git a/README.md b/README.md index 7dc6c7ab..41b03219 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ Run `python -m neural_lam.create_mesh` with suitable options to generate the gra The graphs used for the different models in the [paper](https://arxiv.org/abs/2309.17370) can be created as: * **GC-LAM**: `python -m neural_lam.create_mesh --graph multiscale` -* **Hi-LAM**: `python -m neural_lam.create_mesh --graph hierarchical --hierarchical 1` (also works for Hi-LAM-Parallel) +* **Hi-LAM**: `python -m neural_lam.create_mesh --graph hierarchical --hierarchical` (also works for Hi-LAM-Parallel) * **L1-LAM**: `python -m neural_lam.create_mesh --graph 1level --levels 1` The graph-related files are stored in a directory called `graphs`. diff --git a/neural_lam/create_mesh.py b/neural_lam/create_mesh.py index 40f7ba0e..21b8bf6e 100644 --- a/neural_lam/create_mesh.py +++ b/neural_lam/create_mesh.py @@ -169,10 +169,9 @@ def main(input_args=None): ) parser.add_argument( "--plot", - type=int, - default=0, + action="store_true", help="If graphs should be plotted during generation " - "(default: 0 (false))", + "(default: False)", ) parser.add_argument( "--levels", @@ -182,9 +181,8 @@ def main(input_args=None): ) parser.add_argument( "--hierarchical", - type=int, - default=0, - help="Generate hierarchical mesh graph (default: 0, no)", + action="store_true", + help="Generate hierarchical mesh graph (default: False)", ) args = parser.parse_args(input_args) diff --git a/neural_lam/create_parameter_weights.py b/neural_lam/create_parameter_weights.py index 74058d38..4867e609 100644 --- a/neural_lam/create_parameter_weights.py +++ b/neural_lam/create_parameter_weights.py @@ -156,9 +156,8 @@ def main(): ) parser.add_argument( "--distributed", - type=int, - default=0, - help="Run the script in distributed mode (1) or not (0) (default: 0)", + action="store_true", + help="Run the script in distributed mode (default: False)", ) args = parser.parse_args() distributed = bool(args.distributed) diff --git a/neural_lam/train_model.py b/neural_lam/train_model.py index 39f7aecd..c1a6cb89 100644 --- a/neural_lam/train_model.py +++ b/neural_lam/train_model.py @@ -41,10 +41,9 @@ def main(input_args=None): ) parser.add_argument( "--subset_ds", - type=int, - default=0, + action="store_true", help="Use only a small subset of the dataset, for debugging" - "(default: 0=false)", + "(default: false)", ) parser.add_argument( "--seed", type=int, default=42, help="random seed (default: 42)" @@ -71,10 +70,9 @@ def main(input_args=None): ) parser.add_argument( "--restore_opt", - type=int, - default=0, + action="store_true", help="If optimizer state should be restored with model " - "(default: 0 (false))", + "(default: false)", ) parser.add_argument( "--precision", @@ -118,11 +116,10 @@ def main(input_args=None): ) parser.add_argument( "--output_std", - type=int, - default=0, + action="store_true", help="If models should additionally output std.-dev. per " "output dimensions " - "(default: 0 (no))", + "(default: False (no))", ) # Training options @@ -135,10 +132,9 @@ def main(input_args=None): ) parser.add_argument( "--control_only", - type=int, - default=0, + action="store_true", help="Train only on control member of ensemble data " - "(default: 0 (False))", + "(default: False)", ) parser.add_argument( "--loss", @@ -233,7 +229,7 @@ def main(input_args=None): pred_length=args.ar_steps, split="train", subsample_step=args.step_length, - subset=bool(args.subset_ds), + subset=args.subset_ds, control_only=args.control_only, ), args.batch_size, @@ -247,7 +243,7 @@ def main(input_args=None): pred_length=max_pred_length, split="val", subsample_step=args.step_length, - subset=bool(args.subset_ds), + subset=args.subset_ds, control_only=args.control_only, ), args.batch_size, @@ -313,7 +309,7 @@ def main(input_args=None): pred_length=max_pred_length, split="test", subsample_step=args.step_length, - subset=bool(args.subset_ds), + subset=args.subset_ds, ), args.batch_size, shuffle=False, diff --git a/plot_graph.py b/plot_graph.py index 90462194..e47e62c0 100644 --- a/plot_graph.py +++ b/plot_graph.py @@ -38,9 +38,8 @@ def main(): ) parser.add_argument( "--show_axis", - type=int, - default=0, - help="If the axis should be displayed (default: 0 (No))", + action="store_true", + help="If the axis should be displayed (default: False)", ) args = parser.parse_args() diff --git a/tests/test_mllam_dataset.py b/tests/test_mllam_dataset.py index e12a57ae..5c8b7aa1 100644 --- a/tests/test_mllam_dataset.py +++ b/tests/test_mllam_dataset.py @@ -118,7 +118,7 @@ def test_load_reduced_meps_dataset(meps_example_reduced_filepath): def test_create_graph_reduced_meps_dataset(): args = [ "--graph=hierarchical", - "--hierarchical=1", + "--hierarchical", "--data_config=data/meps_example_reduced/data_config.yaml", "--levels=2", ]