chore: allow for cross compilation #50
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # JULEA - Flexible storage framework | |
| # Copyright (C) 2019-2024 Michael Kuhn | |
| # | |
| # This program is free software: you can redistribute it and/or modify | |
| # it under the terms of the GNU Lesser General Public License as published by | |
| # the Free Software Foundation, either version 3 of the License, or | |
| # (at your option) any later version. | |
| # | |
| # This program is distributed in the hope that it will be useful, | |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| # GNU Lesser General Public License for more details. | |
| # | |
| # You should have received a copy of the GNU Lesser General Public License | |
| # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
| name: CI | |
| on: [push, pull_request] | |
| jobs: | |
| style: | |
| name: Style | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| show-progress: false | |
| - name: Check format | |
| run: | | |
| ./scripts/format.sh | |
| - name: Check shell scripts | |
| run: | | |
| ./scripts/check.sh | |
| dependencies: | |
| name: Dependencies | |
| runs-on: ${{ matrix.os.dist }} | |
| timeout-minutes: 180 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - dist: ubuntu-24.04 | |
| compiler: gcc | |
| compiler_version: 13.2.0 | |
| - dist: ubuntu-24.04 | |
| compiler: clang | |
| compiler_version: 18.1.3 | |
| - dist: ubuntu-22.04 | |
| compiler: gcc | |
| compiler_version: 11.4.0 | |
| - dist: ubuntu-22.04 | |
| compiler: clang | |
| compiler_version: 14.0.0 | |
| - dist: ubuntu-20.04 | |
| compiler: gcc | |
| compiler_version: 9.4.0 | |
| - dist: ubuntu-20.04 | |
| compiler: clang | |
| compiler_version: 11.0.0 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| show-progress: false | |
| - name: Cache dependencies | |
| id: cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: dependencies | |
| key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} | |
| - name: Install dependencies | |
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | |
| env: | |
| JULEA_SPACK_COMPILER: ${{ matrix.os.compiler }}@${{ matrix.os.compiler_version }} | |
| run: | | |
| ./scripts/install-dependencies.sh | |
| dependencies-containers: | |
| name: Dependencies (Containers) | |
| runs-on: ubuntu-24.04 | |
| container: ${{ matrix.os.registry }}/${{ matrix.os.image }} | |
| timeout-minutes: 180 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - image: ubuntu:24.04 | |
| registry: docker.io/library | |
| compiler: gcc | |
| compiler_version: 13.2.0 | |
| - image: ubuntu:24.04 | |
| registry: docker.io/library | |
| compiler: clang | |
| compiler_version: 18.1.3 | |
| - image: ubuntu:22.04 | |
| registry: docker.io/library | |
| compiler: gcc | |
| compiler_version: 11.4.0 | |
| - image: ubuntu:22.04 | |
| registry: docker.io/library | |
| compiler: clang | |
| compiler_version: 14.0.0 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| show-progress: false | |
| - name: Install packages | |
| if: ${{ matrix.os.image == 'ubuntu:24.04' || matrix.os.image == 'ubuntu:22.04' }} | |
| run: | | |
| apt update | |
| apt --yes --no-install-recommends install build-essential zstd | |
| apt --yes --no-install-recommends install ca-certificates git gnupg patchelf python3 unzip | |
| if test "${{ matrix.os.compiler }}" = 'clang' | |
| then | |
| apt --yes --no-install-recommends install clang | |
| if test "${{ matrix.os.image }}" = 'ubuntu:24.04' | |
| then | |
| apt --yes --no-install-recommends install libclang-rt-dev | |
| fi | |
| fi | |
| - name: Cache dependencies | |
| id: cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: /dependencies | |
| key: ${{ matrix.os.image }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} | |
| - name: Install dependencies | |
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | |
| env: | |
| JULEA_SPACK_COMPILER: ${{ matrix.os.compiler }}@${{ matrix.os.compiler_version }} | |
| JULEA_SPACK_DIR: /dependencies | |
| run: | | |
| ./scripts/install-dependencies.sh | |
| build: | |
| name: Build | |
| needs: dependencies | |
| runs-on: ${{ matrix.os.dist }} | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - dist: ubuntu-24.04 | |
| compiler: gcc | |
| compiler_version: 13.2.0 | |
| - dist: ubuntu-24.04 | |
| compiler: clang | |
| compiler_version: 18.1.3 | |
| - dist: ubuntu-22.04 | |
| compiler: gcc | |
| compiler_version: 11.4.0 | |
| - dist: ubuntu-22.04 | |
| compiler: clang | |
| compiler_version: 14.0.0 | |
| - dist: ubuntu-20.04 | |
| compiler: gcc | |
| compiler_version: 9.4.0 | |
| - dist: ubuntu-20.04 | |
| compiler: clang | |
| compiler_version: 11.0.0 | |
| dependencies: [system, spack] | |
| exclude: | |
| # libfabric is too old on Ubuntu 20.04 | |
| - os: | |
| dist: ubuntu-20.04 | |
| dependencies: system | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| show-progress: false | |
| - name: Clean up system packages | |
| # We need to remove glib-network, otherwise libproxy might cause crashes. | |
| run: | | |
| sudo apt --yes purge glib-networking | |
| sudo apt --yes --purge autoremove | |
| sudo aa-remove-unknown || true | |
| - name: Install dependencies | |
| if: ${{ matrix.dependencies == 'system' }} | |
| run: | | |
| sudo apt update || true | |
| sudo apt --yes --no-install-recommends install ninja-build pkgconf libglib2.0-dev libbson-dev libfabric-dev libgdbm-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse3-dev libopen-trace-format-dev librados-dev | |
| if test "${{ matrix.os.dist }}" != 'ubuntu-20.04' | |
| then | |
| sudo apt --yes --no-install-recommends install meson | |
| else | |
| sudo apt --yes --no-install-recommends install python3 python3-pip python3-setuptools python3-wheel | |
| sudo pip3 install meson | |
| fi | |
| - name: Cache dependencies | |
| id: cache | |
| if: ${{ matrix.dependencies == 'spack' }} | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: dependencies | |
| key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} | |
| - name: Check dependencies | |
| if: ${{ matrix.dependencies == 'spack' && steps.cache.outputs.cache-hit != 'true' }} | |
| run: | | |
| exit 1 | |
| - name: Configure, build and install | |
| env: | |
| CC: ${{ matrix.os.compiler }} | |
| run: | | |
| ./scripts/ci/build.sh release "${{ matrix.dependencies }}" | |
| tests: | |
| name: Tests | |
| needs: dependencies | |
| runs-on: ${{ matrix.os.dist }} | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - dist: ubuntu-24.04 | |
| compiler: gcc | |
| compiler_version: 13.2.0 | |
| - dist: ubuntu-24.04 | |
| compiler: clang | |
| compiler_version: 18.1.3 | |
| - dist: ubuntu-22.04 | |
| compiler: gcc | |
| compiler_version: 11.4.0 | |
| - dist: ubuntu-22.04 | |
| compiler: clang | |
| compiler_version: 14.0.0 | |
| - dist: ubuntu-20.04 | |
| compiler: gcc | |
| compiler_version: 9.4.0 | |
| - dist: ubuntu-20.04 | |
| compiler: clang | |
| compiler_version: 11.0.0 | |
| dependencies: [system, spack] | |
| julea: | |
| # Default | |
| - object: posix | |
| kv: lmdb | |
| db: sqlite | |
| # Object backends | |
| - object: gio | |
| kv: lmdb | |
| db: sqlite | |
| # KV backends | |
| - object: posix | |
| kv: gdbm | |
| db: sqlite | |
| - object: posix | |
| kv: mongodb | |
| db: sqlite | |
| - object: posix | |
| kv: leveldb | |
| db: sqlite | |
| - object: posix | |
| kv: rocksdb | |
| db: sqlite | |
| - object: posix | |
| kv: sqlite | |
| db: sqlite | |
| # DB backends | |
| - object: posix | |
| kv: lmdb | |
| db: mysql | |
| db-server: mysql | |
| - object: posix | |
| kv: lmdb | |
| db: mysql | |
| db-server: mariadb | |
| exclude: | |
| # libfabric is too old on Ubuntu 20.04 | |
| - os: | |
| dist: ubuntu-20.04 | |
| dependencies: system | |
| services: | |
| mariadb: | |
| image: ${{ matrix.julea.db == 'mysql' && matrix.julea.db-server == 'mariadb' && 'docker.io/library/mariadb' || '' }} | |
| env: | |
| MARIADB_RANDOM_ROOT_PASSWORD: yes | |
| MARIADB_DATABASE: juleadb | |
| MARIADB_USER: julea | |
| MARIADB_PASSWORD: aeluj | |
| ports: | |
| - 3306:3306 | |
| mongodb: | |
| image: ${{ matrix.julea.kv == 'mongodb' && 'docker.io/library/mongo' || '' }} | |
| ports: | |
| - 27017:27017 | |
| mysql: | |
| image: ${{ matrix.julea.db == 'mysql' && matrix.julea.db-server == 'mysql' && 'docker.io/library/mysql' || '' }} | |
| env: | |
| MYSQL_RANDOM_ROOT_PASSWORD: yes | |
| MYSQL_DATABASE: juleadb | |
| MYSQL_USER: julea | |
| MYSQL_PASSWORD: aeluj | |
| ports: | |
| - 3306:3306 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| show-progress: false | |
| - name: Fix mmap_rnd_bits | |
| if: ${{ matrix.os.dist == 'ubuntu-22.04' }} | |
| # FIXME https://github.com/actions/runner-images/issues/9491 | |
| run: | | |
| sudo sysctl vm.mmap_rnd_bits=28 | |
| - name: Clean up system packages | |
| # We need to remove glib-network, otherwise libproxy might cause crashes. | |
| # FIXME https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 | |
| # Ubuntu 20.04 and 22.04 have MySQL 8.0 | |
| run: | | |
| sudo apt --yes purge glib-networking mysql-client mysql-client-8.0 mysql-server mysql-server-8.0 | |
| sudo apt --yes --purge autoremove | |
| sudo aa-remove-unknown || true | |
| sudo rm --recursive --force /var/lib/mysql | |
| - name: Install dependencies | |
| if: ${{ matrix.dependencies == 'system' }} | |
| run: | | |
| sudo apt update || true | |
| sudo apt --yes --no-install-recommends install ninja-build pkgconf libglib2.0-dev libbson-dev libfabric-dev libgdbm-dev liblmdb-dev libsqlite3-dev libleveldb-dev libmongoc-dev libmariadb-dev librocksdb-dev libfuse3-dev libopen-trace-format-dev librados-dev | |
| if test "${{ matrix.os.dist }}" != 'ubuntu-20.04' | |
| then | |
| sudo apt --yes --no-install-recommends install meson | |
| else | |
| sudo apt --yes --no-install-recommends install python3 python3-pip python3-setuptools python3-wheel | |
| sudo pip3 install meson | |
| fi | |
| - name: Cache dependencies | |
| id: cache | |
| if: ${{ matrix.dependencies == 'spack' }} | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: dependencies | |
| key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} | |
| - name: Check dependencies | |
| if: ${{ matrix.dependencies == 'spack' && steps.cache.outputs.cache-hit != 'true' }} | |
| run: | | |
| exit 1 | |
| - name: Configure and build | |
| env: | |
| CC: ${{ matrix.os.compiler }} | |
| run: | | |
| ./scripts/ci/build.sh debug "${{ matrix.dependencies }}" | |
| - name: Create configuration | |
| # FIXME We need to use 127.0.0.1 because localhost tries to use the socket (/tmp/mysql.sock), which does not exist. | |
| run: | | |
| . scripts/environment.sh | |
| JULEA_KV_PATH="/tmp/julea/kv/${{ matrix.julea.kv }}" | |
| if test "${{ matrix.julea.kv }}" = 'mongodb'; then JULEA_KV_PATH='127.0.0.1:juleadb'; fi | |
| JULEA_DB_PATH="/tmp/julea/db/${{ matrix.julea.db }}" | |
| if test "${{ matrix.julea.db }}" = 'mysql'; then JULEA_DB_PATH='127.0.0.1:juleadb:julea:aeluj'; fi | |
| julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend="${{ matrix.julea.object }}" --object-path="/tmp/julea/object/${{ matrix.julea.object }}" --kv-backend="${{ matrix.julea.kv }}" --kv-path="${JULEA_KV_PATH}" --db-backend="${{ matrix.julea.db }}" --db-path="${JULEA_DB_PATH}" | |
| - name: Tests | |
| run: | | |
| . scripts/environment.sh | |
| ./scripts/setup.sh start | |
| ./scripts/test.sh | |
| sleep 10 | |
| ./scripts/test.sh | |
| ./scripts/setup.sh stop | |
| - name: HDF5 Tests | |
| env: | |
| LSAN_OPTIONS: exitcode=0 | |
| run: | | |
| . scripts/environment.sh | |
| ./scripts/setup.sh start | |
| export HDF5_VOL_CONNECTOR=julea-kv | |
| ./scripts/test.sh -r /hdf5 | |
| sleep 10 | |
| ./scripts/test.sh -r /hdf5 | |
| sleep 10 | |
| export HDF5_VOL_CONNECTOR=julea-db | |
| ./scripts/test.sh -r /hdf5 | |
| sleep 10 | |
| ./scripts/test.sh -r /hdf5 | |
| ./scripts/setup.sh stop | |
| - name: Benchmarks | |
| if: ${{ matrix.julea.object == 'posix' && matrix.julea.kv == 'lmdb' && matrix.julea.db == 'sqlite' }} | |
| run: | | |
| . scripts/environment.sh | |
| ./scripts/setup.sh start | |
| ./scripts/benchmark.sh --duration=0 | |
| sleep 10 | |
| ./scripts/benchmark.sh --duration=0 | |
| ./scripts/setup.sh stop | |
| doxygen: | |
| name: Doxygen | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| show-progress: false | |
| - name: Install dependencies | |
| run: | | |
| sudo apt update || true | |
| sudo apt --yes --no-install-recommends install doxygen | |
| - name: Generate documentation | |
| run: | | |
| doxygen | |
| - name: Upload doxygen | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: doxygen | |
| path: html | |
| coverage: | |
| # FIXME Reduce redundancies | |
| # FIXME Increase coverage (currently, only one configuration runs) | |
| name: Coverage | |
| needs: dependencies | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 60 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| show-progress: false | |
| - name: Fix mmap_rnd_bits | |
| # FIXME https://github.com/actions/runner-images/issues/9491 | |
| run: | | |
| sudo sysctl vm.mmap_rnd_bits=28 | |
| - name: Cache dependencies | |
| id: cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: dependencies | |
| key: ubuntu-24.04-gcc-13.2.0-${{ hashFiles('scripts/spack', 'scripts/install-dependencies.sh') }} | |
| - name: Check dependencies | |
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | |
| run: | | |
| exit 1 | |
| - name: Configure and build | |
| env: | |
| CC: gcc | |
| run: | | |
| ./scripts/ci/build.sh coverage spack | |
| - name: Create configuration | |
| run: | | |
| . scripts/environment.sh | |
| julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend=posix --object-path=/tmp/julea/object/posix --kv-backend=lmdb --kv-path=/tmp/julea/kv/lmdb --db-backend=sqlite --db-path=/tmp/julea/db/sqlite | |
| - name: Tests | |
| env: | |
| HDF5_VOL_CONNECTOR: julea-db | |
| LSAN_OPTIONS: exitcode=0 | |
| run: | | |
| . scripts/environment.sh | |
| ./scripts/setup.sh start | |
| ./scripts/test.sh | |
| ./scripts/setup.sh stop | |
| - name: Generate coverage | |
| run: | | |
| . scripts/environment.sh | |
| ninja -C bld coverage | |
| - name: Upload coverage | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage | |
| path: bld/meson-logs/coveragereport | |
| publish: | |
| name: Publish | |
| needs: [doxygen, coverage] | |
| runs-on: ubuntu-24.04 | |
| permissions: | |
| contents: write | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: gh-pages | |
| path: pages | |
| show-progress: false | |
| - name: Download doxygen | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: doxygen | |
| path: doxygen | |
| - name: Download coverage | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: coverage | |
| path: coverage | |
| - name: Prepare | |
| run: | | |
| rsync --archive --human-readable --verbose --delete --exclude=.git doxygen/. pages | |
| mkdir pages/coverage | |
| rsync --archive --human-readable --verbose --delete --exclude=.git coverage/. pages/coverage | |
| touch pages/.nojekyll | |
| - name: Commit | |
| run: | | |
| cd pages | |
| git config user.name 'github-actions[bot]' | |
| git config user.email 'github-actions[bot]@users.noreply.github.com' | |
| git add --all . | |
| git commit --amend --reset-author --message='Pages for ${{ github.sha }}' | |
| git push --force |