ci.yml

# JULEA - Flexible storage framework
# Copyright (C) 2019-2024 Michael Kuhn
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

name: CI
on: [push, pull_request]
jobs:
  style:
    name: Style
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          show-progress: false
      - name: Check format
        run: |
          ./scripts/format.sh
      - name: Check shell scripts
        run: |
          ./scripts/check.sh
  # dependencies:
  #   name: Dependencies
  #   runs-on: ${{ matrix.os.dist }}
  #   timeout-minutes: 180
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       os:
  #         - dist: ubuntu-24.04
  #           compiler: gcc
  #           compiler_version: 13.2.0
  #         - dist: ubuntu-24.04
  #           compiler: clang
  #           compiler_version: 18.1.3
  #         - dist: ubuntu-22.04
  #           compiler: gcc
  #           compiler_version: 11.4.0
  #         - dist: ubuntu-22.04
  #           compiler: clang
  #           compiler_version: 14.0.0
  #         - dist: ubuntu-20.04
  #           compiler: gcc
  #           compiler_version: 9.4.0
  #         - dist: ubuntu-20.04
  #           compiler: clang
  #           compiler_version: 11.0.0
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v4
  #       with:
  #         persist-credentials: false
  #         show-progress: false
  #     - name: Cache dependencies
  #       id: cache
  #       uses: actions/cache@v4
  #       with:
  #         path: dependencies
  #         key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }}
  #     - name: Install dependencies
  #       if: ${{ steps.cache.outputs.cache-hit != 'true' }}
  #       env:
  #         JULEA_SPACK_COMPILER: ${{ matrix.os.compiler }}@${{ matrix.os.compiler_version }}
  #       run: |
  #         ./scripts/install-dependencies.sh
  # dependencies-containers:
  #   name: Dependencies (Containers)
  #   runs-on: ubuntu-24.04
  #   container: ${{ matrix.os.registry }}/${{ matrix.os.image }}
  #   timeout-minutes: 180
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       os:
  #         - image: ubuntu:24.04
  #           registry: docker.io/library
  #           compiler: gcc
  #           compiler_version: 13.2.0
  #         - image: ubuntu:24.04
  #           registry: docker.io/library
  #           compiler: clang
  #           compiler_version: 18.1.3
  #         - image: ubuntu:22.04
  #           registry: docker.io/library
  #           compiler: gcc
  #           compiler_version: 11.4.0
  #         - image: ubuntu:22.04
  #           registry: docker.io/library
  #           compiler: clang
  #           compiler_version: 14.0.0
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v4
  #       with:
  #         persist-credentials: false
  #         show-progress: false
  #     - name: Install packages
  #       if: ${{ matrix.os.image == 'ubuntu:24.04' || matrix.os.image == 'ubuntu:22.04' }}
  #       run: |
  #         apt update
  #         apt --yes --no-install-recommends install build-essential zstd
  #         apt --yes --no-install-recommends install ca-certificates git gnupg patchelf python3 unzip
  #         if test "${{ matrix.os.compiler }}" = 'clang'
  #         then
  #           apt --yes --no-install-recommends install clang

  #           if test "${{ matrix.os.image }}" = 'ubuntu:24.04'
  #           then
  #             apt --yes --no-install-recommends install libclang-rt-dev
  #           fi
  #         fi
  #     - name: Cache dependencies
  #       id: cache
  #       uses: actions/cache@v4
  #       with:
  #         path: /dependencies
  #         key: ${{ matrix.os.image }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }}
  #     - name: Install dependencies
  #       if: ${{ steps.cache.outputs.cache-hit != 'true' }}
  #       env:
  #         JULEA_SPACK_COMPILER: ${{ matrix.os.compiler }}@${{ matrix.os.compiler_version }}
  #         JULEA_SPACK_DIR: /dependencies
  #       run: |
  #         ./scripts/install-dependencies.sh
  # build:
  #   name: Build
  #   needs: dependencies
  #   runs-on: ${{ matrix.os.dist }}
  #   timeout-minutes: 60
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       os:
  #         - dist: ubuntu-24.04
  #           compiler: gcc
  #           compiler_version: 13.2.0
  #         - dist: ubuntu-24.04
  #           compiler: clang
  #           compiler_version: 18.1.3
  #         - dist: ubuntu-22.04
  #           compiler: gcc
  #           compiler_version: 11.4.0
  #         - dist: ubuntu-22.04
  #           compiler: clang
  #           compiler_version: 14.0.0
  #         - dist: ubuntu-20.04
  #           compiler: gcc
  #           compiler_version: 9.4.0
  #         - dist: ubuntu-20.04
  #           compiler: clang
  #           compiler_version: 11.0.0
  #       dependencies: [system, spack]
  #       exclude:
  #         # libfabric is too old on Ubuntu 20.04
  #         - os:
  #             dist: ubuntu-20.04
  #           dependencies: system
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v4
  #       with:
  #         persist-credentials: false
  #         show-progress: false
  #     - name: Clean up system packages
  #       # We need to remove glib-network, otherwise libproxy might cause crashes.
  #       run: |
  #         sudo apt --yes purge glib-networking
  #         sudo apt --yes --purge autoremove
  #         sudo aa-remove-unknown || true
  #     - name: Install dependencies
  #       if: ${{ matrix.dependencies == 'system' }}
  #       run: |
  #         sudo apt update || true
  #         sudo apt --yes --no-install-recommends install ninja-build pkgconf libglib2.0-dev libbson-dev libfabric-dev libgdbm-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse3-dev libopen-trace-format-dev librados-dev
  #         if test "${{ matrix.os.dist }}" != 'ubuntu-20.04'
  #         then
  #           sudo apt --yes --no-install-recommends install meson
  #         else
  #           sudo apt --yes --no-install-recommends install python3 python3-pip python3-setuptools python3-wheel
  #           sudo pip3 install meson
  #         fi
  #     - name: Cache dependencies
  #       id: cache
  #       if: ${{ matrix.dependencies == 'spack' }}
  #       uses: actions/cache/restore@v4
  #       with:
  #         path: dependencies
  #         key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }}
  #     - name: Check dependencies
  #       if: ${{ matrix.dependencies == 'spack' && steps.cache.outputs.cache-hit != 'true' }}
  #       run: |
  #         exit 1
  #     - name: Configure, build and install
  #       env:
  #         CC: ${{ matrix.os.compiler }}
  #       run: |
  #         ./scripts/ci/build.sh release "${{ matrix.dependencies }}"
  
  dependencies: 
    uses: ./.github/workflows/containers.yml
  tests:
    name: Tests
    needs: dependencies
    runs-on: ubuntu-latest
    container: ghcr.io/finnhering/julea-${{ matrix.dependencies }}:${{ matrix.os.dist }}-${{ matrix.os.compiler }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        os:
          - dist: ubuntu-24-04
            compiler: gcc
            compiler_version: 13.2.0
          - dist: ubuntu-24-04
            compiler: clang
            compiler_version: 18.1.3
          - dist: ubuntu-22-04
            compiler: gcc
            compiler_version: 11.4.0
          - dist: ubuntu-22-04
            compiler: clang
            compiler_version: 14.0.0
          - dist: ubuntu-20-04
            compiler: gcc
            compiler_version: 9.4.0
          - dist: ubuntu-20-04
            compiler: clang
            compiler_version: 11.0.0
        dependencies: [system, spack]
        julea:
          # Default
          - object: posix
            kv: lmdb
            db: sqlite
          # Object backends
          - object: gio
            kv: lmdb
            db: sqlite
          # KV backends
          - object: posix
            kv: gdbm
            db: sqlite
          - object: posix
            kv: mongodb
            db: sqlite
          - object: posix
            kv: leveldb
            db: sqlite
          - object: posix
            kv: rocksdb
            db: sqlite
          - object: posix
            kv: sqlite
            db: sqlite
          # DB backends
          - object: posix
            kv: lmdb
            db: mysql
            db-server: mysql
          - object: posix
            kv: lmdb
            db: mysql
            db-server: mariadb
        exclude:
          # libfabric is too old on Ubuntu 20.04
          - os:
              dist: ubuntu-20.04
            dependencies: system
    services:
      mariadb:
        image: ${{ matrix.julea.db == 'mysql' && matrix.julea.db-server == 'mariadb' && 'docker.io/library/mariadb' || '' }}
        env:
          MARIADB_RANDOM_ROOT_PASSWORD: yes
          MARIADB_DATABASE: juleadb
          MARIADB_USER: julea
          MARIADB_PASSWORD: aeluj
        ports:
          - 3306:3306
      mongodb:
        image: ${{ matrix.julea.kv == 'mongodb' && 'docker.io/library/mongo' || '' }}
        ports:
          - 27017:27017
      mysql:
        image: ${{ matrix.julea.db == 'mysql' && matrix.julea.db-server == 'mysql' && 'docker.io/library/mysql' || '' }}
        env:
          MYSQL_RANDOM_ROOT_PASSWORD: yes
          MYSQL_DATABASE: juleadb
          MYSQL_USER: julea
          MYSQL_PASSWORD: aeluj
        ports:
          - 3306:3306
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          show-progress: false
      - name: Fix mmap_rnd_bits
        if: ${{ matrix.os.dist == 'ubuntu-22-04' }}
        # FIXME https://github.com/actions/runner-images/issues/9491
        run: |
          sudo sysctl vm.mmap_rnd_bits=28
      - name: Clean up system packages
        # We need to remove glib-network, otherwise libproxy might cause crashes.
        # FIXME https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263
        # Ubuntu 20.04 and 22.04 have MySQL 8.0
        run: |
          sudo apt --yes purge glib-networking mysql-client mysql-client-8.0 mysql-server mysql-server-8.0
          sudo apt --yes --purge autoremove
          sudo aa-remove-unknown || true
          sudo rm --recursive --force /var/lib/mysql
      # - name: Install dependencies
      #   if: ${{ matrix.dependencies == 'system' }}
      #   run: |
      #     sudo apt update || true
      #     sudo apt --yes --no-install-recommends install ninja-build pkgconf libglib2.0-dev libbson-dev libfabric-dev libgdbm-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse3-dev libopen-trace-format-dev librados-dev
      #     if test "${{ matrix.os.dist }}" != 'ubuntu-20.04'
      #     then
      #       sudo apt --yes --no-install-recommends install meson
      #     else
      #       sudo apt --yes --no-install-recommends install python3 python3-pip python3-setuptools python3-wheel
      #       sudo pip3 install meson
      #     fi
      # - name: Cache dependencies
      #   id: cache
      #   if: ${{ matrix.dependencies == 'spack' }}
      #   uses: actions/cache/restore@v4
      #   with:
      #     path: dependencies
      #     key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }}
      # - name: Check dependencies
      #   if: ${{ matrix.dependencies == 'spack' && steps.cache.outputs.cache-hit != 'true' }}
      #   run: |
      #     exit 1
      - name: Configure and build
        env:
          CC: ${{ matrix.os.compiler }}
        run: |
          ./scripts/ci/build.sh debug "${{ matrix.dependencies }}"
      - name: Create configuration
        # FIXME We need to use 127.0.0.1 because localhost tries to use the socket (/tmp/mysql.sock), which does not exist.
        run: |
          . scripts/environment.sh
          JULEA_KV_PATH="/tmp/julea/kv/${{ matrix.julea.kv }}"
          if test "${{ matrix.julea.kv }}" = 'mongodb'; then JULEA_KV_PATH='127.0.0.1:juleadb'; fi
          JULEA_DB_PATH="/tmp/julea/db/${{ matrix.julea.db }}"
          if test "${{ matrix.julea.db }}" = 'mysql'; then JULEA_DB_PATH='127.0.0.1:juleadb:julea:aeluj'; fi
          julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend="${{ matrix.julea.object }}" --object-path="/tmp/julea/object/${{ matrix.julea.object }}" --kv-backend="${{ matrix.julea.kv }}" --kv-path="${JULEA_KV_PATH}" --db-backend="${{ matrix.julea.db }}" --db-path="${JULEA_DB_PATH}"
      - name: Tests
        run: |
          . scripts/environment.sh
          ./scripts/setup.sh start
          ./scripts/test.sh
          sleep 10
          ./scripts/test.sh
          ./scripts/setup.sh stop
      - name: HDF5 Tests
        env:
          LSAN_OPTIONS: exitcode=0
        run: |
          . scripts/environment.sh
          ./scripts/setup.sh start
          export HDF5_VOL_CONNECTOR=julea-kv
          ./scripts/test.sh -r /hdf5
          sleep 10
          ./scripts/test.sh -r /hdf5
          sleep 10
          export HDF5_VOL_CONNECTOR=julea-db
          ./scripts/test.sh -r /hdf5
          sleep 10
          ./scripts/test.sh -r /hdf5
          ./scripts/setup.sh stop
      - name: Benchmarks
        if: ${{ matrix.julea.object == 'posix' && matrix.julea.kv == 'lmdb' && matrix.julea.db == 'sqlite' }}
        run: |
          . scripts/environment.sh
          ./scripts/setup.sh start
          ./scripts/benchmark.sh --duration=0
          sleep 10
          ./scripts/benchmark.sh --duration=0
          ./scripts/setup.sh stop
  doxygen:
    name: Doxygen
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          show-progress: false
      - name: Install dependencies
        run: |
          sudo apt update || true
          sudo apt --yes --no-install-recommends install doxygen
      - name: Generate documentation
        run: |
          doxygen
      - name: Upload doxygen
        uses: actions/upload-artifact@v4
        with:
          name: doxygen
          path: html
  coverage:
    # FIXME Reduce redundancies
    # FIXME Increase coverage (currently, only one configuration runs)
    name: Coverage
    needs: dependencies
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          show-progress: false
      - name: Fix mmap_rnd_bits
        # FIXME https://github.com/actions/runner-images/issues/9491
        run: |
          sudo sysctl vm.mmap_rnd_bits=28
      - name: Cache dependencies
        id: cache
        uses: actions/cache/restore@v4
        with:
          path: dependencies
          key: ubuntu-24.04-gcc-13.2.0-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }}
      - name: Check dependencies
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        run: |
          exit 1
      - name: Configure and build
        env:
          CC: gcc
        run: |
          ./scripts/ci/build.sh coverage spack
      - name: Create configuration
        run: |
          . scripts/environment.sh
          julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend=posix --object-path=/tmp/julea/object/posix --kv-backend=lmdb --kv-path=/tmp/julea/kv/lmdb --db-backend=sqlite --db-path=/tmp/julea/db/sqlite
      - name: Tests
        env:
          HDF5_VOL_CONNECTOR: julea-db
          LSAN_OPTIONS: exitcode=0
        run: |
          . scripts/environment.sh
          ./scripts/setup.sh start
          ./scripts/test.sh
          ./scripts/setup.sh stop
      - name: Generate coverage
        run: |
          . scripts/environment.sh
          ninja -C bld coverage
      - name: Upload coverage
        uses: actions/upload-artifact@v4
        with:
          name: coverage
          path: bld/meson-logs/coveragereport
  publish:
    name: Publish
    needs: [doxygen, coverage]
    runs-on: ubuntu-24.04
    permissions:
      contents: write
    if: ${{ github.ref == 'refs/heads/master' }}
    timeout-minutes: 30
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: gh-pages
          path: pages
          show-progress: false
      - name: Download doxygen
        uses: actions/download-artifact@v4
        with:
          name: doxygen
          path: doxygen
      - name: Download coverage
        uses: actions/download-artifact@v4
        with:
          name: coverage
          path: coverage
      - name: Prepare
        run: |
          rsync --archive --human-readable --verbose --delete --exclude=.git doxygen/. pages
          mkdir pages/coverage
          rsync --archive --human-readable --verbose --delete --exclude=.git coverage/. pages/coverage
          touch pages/.nojekyll
      - name: Commit
        run: |
          cd pages
          git config user.name 'github-actions[bot]'
          git config user.email 'github-actions[bot]@users.noreply.github.com'
          git add --all .
          git commit --amend --reset-author --message='Pages for ${{ github.sha }}'
          git push --force