Add "datastores" to represent input data from zarr, npy, etc

.github/workflows/ci-pdm-install-and-test-cpu.yml

@@ -39,17 +39,17 @@ jobs:
       - name: Load cache data
         uses: actions/cache/restore@v4
         with:
-          path: data
-          key: ${{ runner.os }}-meps-reduced-example-data-v0.1.0
+          path: tests/datastore_examples/npyfilesmeps/meps_example_reduced.zip
+          key: ${{ runner.os }}-meps-reduced-example-data-v0.2.0
           restore-keys: |
-            ${{ runner.os }}-meps-reduced-example-data-v0.1.0
+            ${{ runner.os }}-meps-reduced-example-data-v0.2.0
 
       - name: Run tests
         run: |
-          pdm run pytest
+          pdm run pytest -vv -s
 
       - name: Save cache data
         uses: actions/cache/save@v4
         with:
-          path: data
-          key: ${{ runner.os }}-meps-reduced-example-data-v0.1.0
+          path: tests/datastore_examples/npyfilesmeps/meps_example_reduced.zip
+          key: ${{ runner.os }}-meps-reduced-example-data-v0.2.0

.github/workflows/ci-pdm-install-and-test-gpu.yml

@@ -44,17 +44,17 @@ jobs:
       - name: Load cache data
         uses: actions/cache/restore@v4
         with:
-          path: data
-          key: ${{ runner.os }}-meps-reduced-example-data-v0.1.0
+          path: tests/datastore_examples/npyfilesmeps/meps_example_reduced.zip
+          key: ${{ runner.os }}-meps-reduced-example-data-v0.2.0
           restore-keys: |
-            ${{ runner.os }}-meps-reduced-example-data-v0.1.0
+            ${{ runner.os }}-meps-reduced-example-data-v0.2.0
 
       - name: Run tests
         run: |
-          pdm run pytest
+          pdm run pytest -vv -s
 
       - name: Save cache data
         uses: actions/cache/save@v4
         with:
-          path: data
-          key: ${{ runner.os }}-meps-reduced-example-data-v0.1.0
+          path: tests/datastore_examples/npyfilesmeps/meps_example_reduced.zip
+          key: ${{ runner.os }}-meps-reduced-example-data-v0.2.0

.github/workflows/ci-pip-install-and-test-cpu.yml

@@ -29,17 +29,17 @@ jobs:
       - name: Load cache data
         uses: actions/cache/restore@v4
         with:
-          path: data
-          key: ${{ runner.os }}-meps-reduced-example-data-v0.1.0
+          path: tests/datastore_examples/npyfilesmeps/meps_example_reduced.zip
+          key: ${{ runner.os }}-meps-reduced-example-data-v0.2.0
           restore-keys: |
-            ${{ runner.os }}-meps-reduced-example-data-v0.1.0
+            ${{ runner.os }}-meps-reduced-example-data-v0.2.0
 
       - name: Run tests
         run: |
-          python -m pytest
+          python -m pytest -vv -s
 
       - name: Save cache data
         uses: actions/cache/save@v4
         with:
-          path: data
-          key: ${{ runner.os }}-meps-reduced-example-data-v0.1.0
+          path: tests/datastore_examples/npyfilesmeps/meps_example_reduced.zip
+          key: ${{ runner.os }}-meps-reduced-example-data-v0.2.0

.github/workflows/ci-pip-install-and-test-gpu.yml

@@ -34,17 +34,17 @@ jobs:
       - name: Load cache data
         uses: actions/cache/restore@v4
         with:
-          path: data
-          key: ${{ runner.os }}-meps-reduced-example-data-v0.1.0
+          path: tests/datastore_examples/npyfilesmeps/meps_example_reduced.zip
+          key: ${{ runner.os }}-meps-reduced-example-data-v0.2.0
           restore-keys: |
-            ${{ runner.os }}-meps-reduced-example-data-v0.1.0
+            ${{ runner.os }}-meps-reduced-example-data-v0.2.0
 
       - name: Run tests
         run: |
-          python -m pytest
+          python -m pytest -vv -s
 
       - name: Save cache data
         uses: actions/cache/save@v4
         with:
-          path: data
-          key: ${{ runner.os }}-meps-reduced-example-data-v0.1.0
+          path: tests/datastore_examples/npyfilesmeps/meps_example_reduced.zip
+          key: ${{ runner.os }}-meps-reduced-example-data-v0.2.0

.github/workflows/pre-commit.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11"]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python

.gitignore

@@ -1,14 +1,14 @@
 ### Project Specific ###
 wandb
-slurm_log*
 saved_models
 lightning_logs
 data
 graphs
 *.sif
 sweeps
-test_*.sh
 .vscode
+*.html
+*.zarr
 *slurm*
 
 ### Python ###
@@ -75,8 +75,15 @@ tags
 
 # Coc configuration directory
 .vim
+.vscode
+
+# macos
+.DS_Store
+__MACOSX
 
 # pdm (https://pdm-project.org/en/stable/)
 .pdm-python
+.venv
+
 # exclude pdm.lock file so that both cpu and gpu versions of torch will be accepted by pdm
 pdm.lock

CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [unreleased](https://github.com/joeloskarsson/neural-lam/compare/v0.2.0...HEAD)
 
+### Added
+
+- Introduce Datastores to represent input data from different sources, including zarr and numpy.
+  [\#66](https://github.com/mllam/neural-lam/pull/66)
+ @leifdenby @sadamov
+
 ## [v0.2.0](https://github.com/joeloskarsson/neural-lam/releases/tag/v0.2.0)
 
 ### Added