Merge pull request #72 from climate-resource/update-structure

Update repo structure

.github/ISSUE_TEMPLATE/bug.md

@@ -35,7 +35,6 @@ def test_bug_12():
 
  - OS: [e.g. Windows, Linux, macOS]
  - Python version [e.g. Python 3.11]
- - Please also upload your `poetry.lock` file (first run `poetry lock` to make sure the lock file is up-to-date)
 
 ## Additional context
 <!--- Add any other context about the problem here. -->

.github/workflows/bump.yaml

@@ -5,7 +5,7 @@ on:
     inputs:
       bump_rule:
         type: choice
-        description: How to bump the project's version (see https://python-poetry.org/docs/cli/#version)
+        description: How to bump the project's version (see https://github.com/carstencodes/pdm-bump#usage)
         options:
           - no-pre-release
           # no micro because we always sit on a pre-release in main,
@@ -41,7 +41,6 @@ jobs:
         run: |
           pdm self add pdm-bump
 
-      # Install the package and towncrier with pixi
       - uses: ./.github/actions/setup
         with:
           pixi-environments: "all-dev"
@@ -72,15 +71,13 @@ jobs:
           # version as the tagged commit)
           BASE_VERSION=`sed -ne 's/^version = "\([0-9\.a]*\)"/\1/p' pyproject.toml`
 
-          # Bump (need two commands because of a bug in pdm-bump)
-          # See https://github.com/carstencodes/pdm-bump/issues/326
-          pdm bump micro
+          # Bump to pre-release of next version
           pdm bump pre-release --pre alpha
 
           NEW_VERSION=`sed -ne 's/^version = "\([0-9\.a]*\)"/\1/p' pyproject.toml`
           echo "Bumping version $BASE_VERSION > $NEW_VERSION"
 
-          # Skip pre-commit to avoid pixi install throwing weird errors
           # Commit and push
+          # Skip pre-commit to avoid pixi install throwing weird errors
           git commit -n -a -m "bump(pre-release): version $BASE_VERSION > $NEW_VERSION"
           git push

.github/workflows/ci.yaml

@@ -17,7 +17,8 @@ jobs:
         with:
           pixi-environments: "all-dev"
       - name: mypy
-        run: MYPYPATH=stubs pixi run --frozen -e all-dev mypy src
+        run: |
+          MYPYPATH=stubs pixi run --frozen -e all-dev mypy src
 
   docs:
     if: ${{ !github.event.pull_request.draft }}
@@ -30,8 +31,10 @@ jobs:
           pixi-environments: "docs"
       - name: docs
         run: |
-          pixi run --frozen -e docs typer input4mips_validation.cli utils docs --output docs/cli/index.md --name input4mips-validation-cli
+          pixi run --frozen -e docs typer input4mips_validation.cli utils docs --output docs/cli/index.md --name input4mips-validation
           pixi run --frozen -e docs mkdocs build --strict
+      - name: docs-with-changelog
+        run: |
           # Check CHANGELOG will build too
           pixi run --frozen -e all-dev towncrier build --yes
           pixi run --frozen -e all-dev mkdocs build --strict
@@ -47,22 +50,32 @@ jobs:
     runs-on: "${{ matrix.os }}"
     defaults:
       run:
-        # This might be needed for Windows and doesn't seem to affect unix-based systems
-        # so we include it. If you have better proof of whether this is needed or not,
+        # This might be needed for Windows
+        # and doesn't seem to affect unix-based systems so we include it.
+        # If you have better proof of whether this is needed or not,
         # feel free to update.
         shell: bash
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
       - uses: ./.github/actions/setup
         with:
+          # When running the tests, install with all optional dependencies
+          # to get maximum test coverage.
+          # If we find that we're getting failures
+          # when people try to run without installing optional dependencies,
+          # we should add a CI step that runs the tests without optional dependencies too.
+          # We don't have that right now, because we're not sure this pain point exists.
+          # (Although pixi actually controls this, rather than the flags passed here.)
           pixi-environments: "test-${{ matrix.test-python-id }}"
       - name: Run tests
         run: |
           pixi run --frozen -e "test-${{ matrix.test-python-id }}" pytest -r a -v src tests --doctest-modules --cov=src --cov-report=term-missing --cov-report=xml
-          pixi run --frozen -e "test-${{ matrix.test-python-id }}"  coverage report
-      - name: Upload coverage reports to Codecov
-        uses: codecov/codecov-action@v3
+          pixi run --frozen -e "test-${{ matrix.test-python-id }}" coverage report
+      - name: Upload coverage reports to Codecov with GitHub Action
+        uses: codecov/codecov-action@v4.2.0
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   imports-without-extras:
     strategy:
@@ -116,13 +129,11 @@ jobs:
   #       uses: actions/checkout@v4
   #     - uses: ./.github/actions/setup
   #       with:
-  #         python-version: "3.9"
-  #         venv-id: "licence-check"
-  #         poetry-dependency-install-flags: "--all-extras"
+  #         pixi-environments: "all-dev"
   #     - name: Check licences of dependencies
   #       shell: bash
   #       run: |
   #         TEMP_FILE=$(mktemp)
-  #         poetry export --without=tests --without=docs --without=dev > $TEMP_FILE
-  #         poetry run liccheck -r $TEMP_FILE -R licence-check.txt
+  #         pdm export --prod > $TEMP_FILE
+  #         pixi run --frozen -e all-dev liccheck -r $TEMP_FILE -R licence-check.txt
   #         cat licence-check.txt

.github/workflows/deploy.yaml

@@ -11,6 +11,10 @@ defaults:
 jobs:
   deploy-pypi:
     name: Deploy to PyPI
+    # Having an environment for deployment is strongly recommend by PyPI
+    # https://docs.pypi.org/trusted-publishers/adding-a-publisher/#github-actions
+    # You can comment this line out if you don't want it.
+    environment: deploy
     runs-on: ubuntu-latest
     permissions:
       # this permission is mandatory for trusted publishing with PyPI

.github/workflows/install-pypi.yaml

@@ -63,11 +63,20 @@ jobs:
       if: matrix.os != 'windows-latest'
       run: |
         if grep -q "WARN" stderr.txt; then echo "Warnings in pip install output" && cat stderr.txt && exit 1; else exit 0; fi
-    - name: Get version
+    - name: Get version non-windows
+      if: matrix.os != 'windows-latest'
       run: |
         INSTALLED_VERSION=`python -c 'import input4mips_validation; print(f"v{input4mips_validation.__version__}")'`
         echo $INSTALLED_VERSION
         echo "INSTALLED_VERSION=$INSTALLED_VERSION" >> $GITHUB_ENV
+    - name: Get version windows
+      if: matrix.os == 'windows-latest'
+      run: |
+        chcp 65001  # use utf-8
+        python -c 'import input4mips_validation; f = open("version.txt", "w"); f.write(f"INSTALLED_VERSION=v{input4mips_validation.__version__}"); f.close()'
+        echo "Showing version.txt"
+        type version.txt
+        type version.txt >> $env:GITHUB_ENV
     - name: Checkout repository
       uses: actions/checkout@v4
       with:
@@ -87,4 +96,5 @@ jobs:
       run: |
         export LD_LIBRARY_PATH="$HOME/udunits/lib/:$LD_LIBRARY_PATH"
         python -c 'import ctypes.util; print(ctypes.util.find_library("udunits2"))'
+        # Can't test coverage here because paths are different
         pytest tests -r a -vv -s

.gitignore

@@ -1,35 +1,35 @@
-# Big test data. Not sure whether to add this or just store in a separate
-# test data repo or something (or just download every time...)
-tests/test-data/input4MIPs/CMIP6Plus/CMIP/PCMDI
-
 # Default data download directory
 src/input4mips_validation/cvs_handling/input4MIPs/user_cvs
 src/input4mips_validation/cvs/user_cvs
 src/input4mips_validation/cvs/input4MIPs_CVs_*
 
-# Auto-generated docs and helper files
-docs/api
-docs/cli
+# Auto-generated docs stuff
 docs/how-to-guides/**/*.nc
 
+# pixi stuff
+.pixi
+
 # Notebooks
 *.ipynb
 
-# Lock file
-poetry.lock
+# Auto-generated docs and helper files
+docs/api/*
+!docs/api/.gitkeep
+docs/cli/*
+!docs/cli/.gitkeep
 
 # pdm stuff
 .pdm-python
 
-# pixi stuff
-.pixi
-
 # Databases
 *.db
 
 # Jupyter cache
 .jupyter_cache
 
+# IDE stuff
+.idea/
+
 # Ruff cache
 .ruff_cache
 

.pre-commit-config.yaml

@@ -8,15 +8,14 @@ ci:
   skip: [
     pixi-install,
     environment-docs,
-    pdm-export,
     pdm-lock-check,
     auto-generated-classes,
   ]
 
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: 'v4.5.0'
+    rev: "v4.5.0"
     hooks:
       - id: check-added-large-files
       - id: check-ast
@@ -40,7 +39,7 @@ repos:
         language: fail
         files: "\\.rej$"
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.1.8'
+    rev: "v0.1.8"
     hooks:
       - id: ruff
         args: [ --fix, --exit-non-zero-on-fix ]
@@ -50,13 +49,11 @@ repos:
     hooks:
       # Check that the lock file is up to date.
       # We need the pdm lock file too
-      # so that we can build locked version of the package.
+      # so that we can build the locked version of the package.
       - id: pdm-lock-check
-        args: [ --dev, --group, ":all" ]
-      # Put a requirements.txt file in the repo too
-      - id: pdm-export
-        args: ['--prod', '-o', 'requirements.txt', '--without-hashes']
-      # - id: pdm-sync  # not used here because we manage environments with pixi
+        args: ["--dev", "-G", ":all", "--strategy", "inherit_metadata" ]
+      # No requirements exported because we use pixi
+      # and conda doesn't play nice with pip anyway.
   - repo: local
     hooks:
       # pixi doesn't have a lock/check command yet,

.readthedocs.yaml

@@ -2,8 +2,6 @@
 # Read the Docs configuration file
 # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
 
-# Thank you also https://browniebroke.com/blog/specify-docs-dependency-groups-with-poetry-and-read-the-docs/
-
 # Required
 version: 2
 
@@ -15,9 +13,8 @@ build:
   jobs:
     pre_build:
       - pip install --no-deps .
-      - typer input4mips_validation.cli utils docs --output docs/cli/index.md --name input4mips-validation-cli
-      - ls
       - input4mips-validation --help
+      - typer input4mips_validation.cli utils docs --output docs/cli/index.md --name input4mips-validation
       # Can add other checks in here if things blow up confusingly
     post_install:
       - mamba list

Makefile

@@ -32,55 +32,58 @@ checks:  ## run all the linting checks of the codebase
 ruff-fixes:  ## fix the code using ruff
     # format before and after checking so that the formatted stuff is checked and
     # the fixed stuff is formatted
-	pixi run -e all-dev ruff format src tests scripts
-	pixi run -e all-dev ruff check src tests scripts --fix
-	pixi run -e all-dev ruff format src tests scripts
+	pixi run -e all-dev ruff format src tests scripts docs
+	pixi run -e all-dev ruff check src tests scripts docs --fix
+	pixi run -e all-dev ruff format src tests scripts docs
 
 
 .PHONY: test
 test:  ## run the tests
 	pixi run -e all-dev pytest src tests -r a -v --doctest-modules --cov=src
 
 # Note on code coverage and testing:
-# You must specify cov=src as otherwise funny things happen when doctests are
-# involved.
-# If you want to debug what is going on with coverage, we have found
-# that adding COVERAGE_DEBUG=trace to the front of the below command
-# can be very helpful as it shows you if coverage is tracking the coverage
+# You must specify cov=src.
+# Otherwise, funny things happen when doctests are involved.
+# If you want to debug what is going on with coverage,
+# we have found that adding COVERAGE_DEBUG=trace
+# to the front of the below command
+# can be very helpful as it shows you
+# if coverage is tracking the coverage
 # of all of the expected files or not.
-# We are sure that the coverage maintainers would appreciate a PR that improves
-# the coverage handling when there are doctests and a `src` layout like ours.
+# We are sure that the coverage maintainers would appreciate a PR
+# that improves the coverage handling when there are doctests
+# and a `src` layout like ours.
+
+docs/cli/index.md: src/input4mips_validation/cli/__init__.py  ## auto-generate the typer app docs
+	pixi run -e all-dev typer input4mips_validation.cli utils docs --output docs/cli/index.md --name input4mips-validation
 
 .PHONY: docs
 docs: docs/cli/index.md  ## build the docs
 	pixi run -e all-dev mkdocs build
 
-.PHONY: docs
+.PHONY: docs-strict
 docs-strict: docs/cli/index.md  ## build the docs strictly (e.g. raise an error on warnings, this most closely mirrors what we do in the CI)
 	pixi run -e all-dev mkdocs build --strict
 
 .PHONY: docs-serve
 docs-serve: docs/cli/index.md  ## serve the docs locally
 	pixi run -e all-dev mkdocs serve
 
-docs/cli/index.md: src/input4mips_validation/cli/__init__.py  ## auto-generate the typer app docs
-	pixi run -e all-dev typer input4mips_validation.cli utils docs --output docs/cli/index.md --name input4mips-validation-cli
-
 .PHONY: changelog-draft
 changelog-draft:  ## compile a draft of the next changelog
-	pixi run -e all-dev towncrier build --draft
+	pixi run -e all-dev towncrier build --draft --version draft
 
 .PHONY: licence-check
 licence-check:  ## Check that licences of the dependencies are suitable
 	# Will likely fail on Windows, but Makefiles are in general not Windows
 	# compatible so we're not too worried
-	pdm export -o $(TEMP_FILE) --without=tests --without=docs --without=dev
-	pdm run liccheck -r $(TEMP_FILE) -R licence-check.txt
-	rm $(TEMP_FILE)
+	pdm export --without=tests --without=docs --without=dev > $(TEMP_FILE)
+	pixi run -e all-dev liccheck -r $(TEMP_FILE) -R licence-check.txt
+	rm -f $(TEMP_FILE)
 
 .PHONY: virtual-environment
 virtual-environment:  ## update virtual environment, create a new one if it doesn't already exist
 	pixi install -e all-dev
 	pixi run -e all-dev pre-commit install
 	# Make sure pdm lock file is up to date too
-	pdm lock --strategy=inherit_metadata --dev --group :all
+	pdm lock --dev --group :all --strategy inherit_metadata

README.md

@@ -184,7 +184,7 @@ Hence, you will also need to make sure that pdm is installed on your system
 although we found that installing with [pipx](https://pipx.pypa.io/stable/installation/)
 worked perfectly for us).
 
-For all of work, we use our `Makefile`.
+For all of our work, we use our `Makefile`.
 You can read the instructions out and run the commands by hand if you wish,
 but we generally discourage this because it can be error prone.
 In order to create your environment, run `make virtual-environment`.

changelog/72.trivial.md

@@ -0,0 +1,9 @@
+Updated a number of internal set up bits and pieces, e.g.:
+
+- removing the mention of poetry from the bug issue template
+- updating the CI workflows
+- using an environment for deploying to PyPI
+- updating the `Makefile`
+- updating the docs to use a `NAVIGATION.md` file with literate-nav
+- fixing all the pull request links in the CHANGELOG in the docs
+- simplifying the auto-generated docs