Gitlab CI, Docker, src layout

.gitlab-ci.yml

@@ -1,10 +1,124 @@
+stages:
+  # - image-pixi
+  # - pixi
+  - image
+  - lint
+  - test
+  - doc
+  - build
+
 variables:
-  CODECOV_TOKEN: 0ac60028-17ba-4383-b4ad-b5ba5ca35f3d
+  COVERAGE_DIR: .coverage_$CI_COMMIT_SHA
+  DOCKER_IMAGE_PATH: registry.heptapod.net:443/fluiddyn/fluidimage/ci/default
+
+image: $DOCKER_IMAGE_PATH:stable
+
+# ugly workaround https://gitlab.com/gitlab-org/gitlab/-/issues/370052#note_1207556577
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
+      when: never
+    - if: $CI_COMMIT_BRANCH
+    - if: $CI_COMMIT_TAG
+
+
+# Build an image for the other tasks; this should be a scheduled job, as
+# it is quite unnecessary to run on every invocation.
+image:build:
+  stage: image
+  needs: []
+  tags:
+    - container-registry-push
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+    - if: '$CI_BUILD_IMAGES == "1"'
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [ "" ]
+  before_script:
+    - ""
+  script:
+    - |
+      cat > /kaniko/.docker/config.json <<EOF
+      {
+        "auths": {
+          "$CI_REGISTRY": {
+            "username": "$CI_REGISTRY_USER",
+            "password": "$CI_REGISTRY_PASSWORD"
+          }
+        }
+      }
+      EOF
+    - >
+      /kaniko/executor --context $CI_PROJECT_DIR
+      --dockerfile $CI_PROJECT_DIR/docker/Dockerfile
+      --single-snapshot
+      --cleanup
+      --destination $DOCKER_IMAGE_PATH:stable
+
+
+validate_code:
+  stage: lint
+  needs:
+    - job: "image:build"
+      optional: true
+  script:
+    - nox -s validate_code
+
 
 tests:
-  image: fluiddyn/python3-stable:testing
+  stage: test
+  needs:
+    - job: "image:build"
+      optional: true
+  script:
+    - xvfb-run nox -s test-cov
+
+
+pages:
+  stage: doc
+  needs:
+    - job: "image:build"
+      optional: true
+  variables:
+    FLUIDFFT_TRANSONIC_BACKEND: "python"
+  script:
+    - python -m venv .venv
+    - . .venv/bin/activate
+    - pip install fluidfft --no-deps
+    - pdm sync -G doc -G fft -G test --no-self
+    - pip install . --config-settings=setup-args=-Dtransonic-backend=python --no-deps
+    - pdm run xvfb-run --auto-servernum sphinx-build -W -b html -d doc/_build/doctrees doc doc/_build/html
+    - mkdir -p public/$CI_COMMIT_REF_NAME
+    - rsync -rvc --delete doc/_build/html/* public/$CI_COMMIT_REF_NAME/
+    # This directory can become too large leading to error.
+    # It can be purged with the botton "Clear runner caches"
+    # in https://foss.heptapod.net/fluiddyn/fluidsim/-/pipelines
+    - ls public
+    - echo "CI_COMMIT_REF_NAME="$CI_COMMIT_REF_NAME
+    - echo See https://fluiddyn.pages.heptapod.net/fluidsim/$CI_COMMIT_REF_NAME
+  artifacts:
+    name: "$CI_COMMIT_REF_NAME"
+    paths:
+      - public
+    expire_in: 5 days
+  when: always
+
+
+build:package:
+  stage: build
+  before_script:
+    - pip install build twine
   script:
-    - python -m venv venv
-    - source venv/bin/activate
-    - pip install -U pip nox
-    - xvfb-run nox -s tests-cov
+    - cd lib
+    - rm -rf dist
+    - python -m build
+    - ls dist
+    - twine check --strict dist/*
+  needs: []
+  artifacts:
+    when: always
+    paths:
+      - lib/dist
+    expire_in: 24 hrs

docker/Dockerfile

@@ -0,0 +1,44 @@
+FROM python:3.9
+LABEL Pierre Augier <pierre.augier@univ-grenoble-alpes.fr>
+
+RUN apt-get update
+
+# also include utilities for debugging
+RUN apt-get install -y --no-install-recommends \
+  clang \
+  emacs vim \
+  meld less \
+  # python3-pip is for Mercurial (hg-git, hg-evolve)
+  mercurial python3-pip git
+RUN apt-get install -y --no-install-recommends \
+  rsync \
+  libgl1 xvfb xauth libgl1-mesa-dev \
+  libxkbcommon-x11-0 \
+  sudo graphviz
+
+RUN rm -rf /var/lib/apt/lists/*
+
+RUN groupadd -g 1000 appuser && useradd -m -r -u 1000 -g appuser -s /bin/bash appuser -s /bin/bash && usermod -a -G sudo appuser
+RUN echo appuser:appuser | chpasswd
+USER appuser
+ARG HOME=/home/appuser
+RUN mkdir -p $HOME/opt
+WORKDIR $HOME/opt
+RUN echo $USER $HOME $PWD && whoami
+
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
+RUN $(hg debuginstall -T '{pythonexe}') -m pip install hg-evolve hg-git --no-cache-dir --user
+
+COPY --chown=appuser:appuser docker/hgrc $HOME/.hgrc
+COPY --chown=appuser:appuser docker/pythranrc $HOME/.pythranrc
+
+RUN mkdir -p $HOME/.local/include
+RUN mkdir -p $HOME/.local/lib
+
+ENV PATH=$HOME/.local/bin:$PATH
+
+RUN mkdir -p $HOME/.config/matplotlib
+RUN echo 'backend      : agg' > $HOME/.config/matplotlib/matplotlibrc
+
+RUN python -m pip install -U pip pdm nox --user

docker/Makefile

@@ -0,0 +1,49 @@
+image := image-fluidsim
+tag := $(shell date -I'date'| tr -d "[:punct:]")
+
+define STR_HELP
+This makefile can be used to build images and start containers.
+
+Examples
+--------
+$ make start  # starts Docker service
+$ make pull   # pulls a Docker image fron Docker hub
+$ make build  # by default builds python3-stable image
+$ make run
+$ make build image=python3-stable
+$ make cleanall  # clean (unnecessary, not all) containers and images
+
+endef
+
+export STR_HELP
+
+.PHONY: help
+
+help:
+	@echo "$$STR_HELP"
+
+start:
+	systemctl start docker
+
+build:
+	docker build -f Dockerfile -t fluiddyn/$(image) ..
+	docker tag fluiddyn/$(image) fluiddyn/$(image):$(tag)
+
+list:
+	@printf "\nImages: "
+	docker images
+	@printf "\nContainers: "
+	docker ps
+
+cleancontainers:
+	@echo "Clean exited containers."
+	for cont in $$(docker ps -a | awk 'NR>1{print $$1}'); do docker stop $$cont; docker rm $$cont; done
+
+cleanimages:
+	@echo "Clean dangling images with no tag."
+	for img in $$(docker images -qf "dangling=true"); do docker rmi -f $$img; done
+
+cleanall: cleancontainers cleanimages cleanmako
+
+run:
+	docker run --name $(image) --restart always -it fluiddyn/$(image) bash

docker/hgrc

@@ -0,0 +1,34 @@
+# example user config (see 'hg help config' for more info)
+[ui]
+# name and email, e.g.
+# username = Jane Doe <jdoe@example.com>
+username = appuser
+editor = emacs -nw -Q
+
+# We recommend enabling tweakdefaults to get slight improvements to
+# the UI over time. Make sure to set HGPLAIN in the environment when
+# writing scripts!
+tweakdefaults = True
+
+[extensions]
+# uncomment these lines to enable some popular extensions
+# (see 'hg help extensions' for more info)
+#
+churn =
+shelve =
+hgext.extdiff =
+rebase =
+absorb =
+evolve =
+topic =
+hggit =
+
+[extdiff]
+cmd.meld =
+
+[trusted]
+users = root
+
+[alias]
+lg = log -G
+up = up -v

docker/pythranrc

@@ -0,0 +1,7 @@
+[pythran]
+complex_hook = True
+
+[compiler]
+CXX=clang++
+CC=clang
+blas=openblas

meson.build

@@ -65,4 +65,4 @@ if use_pythran
 
 endif
 
-subdir('fluidimage')
+subdir('src/fluidimage')

pyproject.toml

@@ -65,6 +65,7 @@ setup = ['-Doptimization=3']
 
 [tool.pdm]
 distribution = true
+package-dir = "src"
 ignore_package_warnings = ["ipython"]
 
 [tool.pdm.dev-dependencies]
@@ -99,9 +100,9 @@ dev = [
 ]
 
 [tool.pdm.scripts]
-black = 'black fluidsim doc'
-black_check = 'black --check fluidsim doc'
-lint = {shell="pylint -rn --rcfile=pylintrc --jobs=$(nproc) src doc tests plugins --exit-zero"}
+black = 'black src doc'
+black_check = 'black --check src doc'
+lint = {shell="pylint -rn --rcfile=pylintrc --jobs=$(nproc) src doc --exit-zero"}
 validate_code = {composite = ["black_check", "lint"]}
 
 [tool.pdm.options]
@@ -111,7 +112,7 @@ install = ["--no-isolation"]
 
 
 [tool.coverage.run]
-source = ["./fluidimage"]
+source = ["./src"]
 data_file = ".coverage/coverage"
 omit = [
     "**/__pythran__/*.py",
@@ -148,4 +149,4 @@ extend-exclude = "/(__pythran__|__python__|__numba__|build|doc/_build|\\.ipynb_c
 [tool.isort]
 profile = "black"
 line_length = 82
-src_paths = ["fluidsim", "doc"]
+src_paths = ["src", "doc"]

fluidimage/meson.build → src/fluidimage/meson.build

@@ -2,7 +2,6 @@
 python_sources = [
   '__init__.py',
   '_opencv.py',
-  '_path_image_samples.py',
   '_version.py',
   'config.py',
   'run_from_xml.py',