fixup #4095
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
--- | |
name: tests | |
on: | |
workflow_dispatch: | |
push: | |
branches-ignore: | |
- 'development/**' | |
- 'q/*/**' | |
env: | |
# Secrets | |
azurebackend_AZURE_STORAGE_ACCESS_KEY: >- | |
${{ secrets.AZURE_STORAGE_ACCESS_KEY }} | |
azurebackend_AZURE_STORAGE_ACCOUNT_NAME: >- | |
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }} | |
azurebackend_AZURE_STORAGE_ENDPOINT: >- | |
${{ secrets.AZURE_STORAGE_ENDPOINT }} | |
azurebackend2_AZURE_STORAGE_ACCESS_KEY: >- | |
${{ secrets.AZURE_STORAGE_ACCESS_KEY_2 }} | |
azurebackend2_AZURE_STORAGE_ACCOUNT_NAME: >- | |
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME_2 }} | |
azurebackend2_AZURE_STORAGE_ENDPOINT: >- | |
${{ secrets.AZURE_STORAGE_ENDPOINT_2 }} | |
azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY: >- | |
${{ secrets.AZURE_STORAGE_ACCESS_KEY }} | |
azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME: >- | |
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }} | |
azurebackendmismatch_AZURE_STORAGE_ENDPOINT: >- | |
${{ secrets.AZURE_STORAGE_ENDPOINT }} | |
azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY: >- | |
${{ secrets.AZURE_STORAGE_ACCESS_KEY }} | |
azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME: >- | |
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }} | |
azurenonexistcontainer_AZURE_STORAGE_ENDPOINT: >- | |
${{ secrets.AZURE_STORAGE_ENDPOINT }} | |
azuretest_AZURE_BLOB_ENDPOINT: "${{ secrets.AZURE_STORAGE_ENDPOINT }}" | |
b2backend_B2_ACCOUNT_ID: "${{ secrets.B2BACKEND_B2_ACCOUNT_ID }}" | |
b2backend_B2_STORAGE_ACCESS_KEY: >- | |
${{ secrets.B2BACKEND_B2_STORAGE_ACCESS_KEY }} | |
GOOGLE_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}" | |
GOOGLE_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}" | |
AWS_S3_BACKEND_ACCESS_KEY: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY }}" | |
AWS_S3_BACKEND_SECRET_KEY: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY }}" | |
AWS_S3_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY_2 }}" | |
AWS_S3_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY_2 }}" | |
AWS_GCP_BACKEND_ACCESS_KEY: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY }}" | |
AWS_GCP_BACKEND_SECRET_KEY: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY }}" | |
AWS_GCP_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY_2 }}" | |
AWS_GCP_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY_2 }}" | |
b2backend_B2_STORAGE_ENDPOINT: "${{ secrets.B2BACKEND_B2_STORAGE_ENDPOINT }}" | |
gcpbackend2_GCP_SERVICE_EMAIL: "${{ secrets.GCP2_SERVICE_EMAIL }}" | |
gcpbackend2_GCP_SERVICE_KEY: "${{ secrets.GCP2_SERVICE_KEY }}" | |
gcpbackend2_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey | |
gcpbackend_GCP_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}" | |
gcpbackend_GCP_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}" | |
gcpbackendmismatch_GCP_SERVICE_EMAIL: >- | |
${{ secrets.GCPBACKENDMISMATCH_GCP_SERVICE_EMAIL }} | |
gcpbackendmismatch_GCP_SERVICE_KEY: >- | |
${{ secrets.GCPBACKENDMISMATCH_GCP_SERVICE_KEY }} | |
gcpbackend_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey | |
gcpbackendmismatch_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey | |
gcpbackendnoproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey | |
gcpbackendproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey | |
# Configs | |
ENABLE_LOCAL_CACHE: "true" | |
REPORT_TOKEN: "report-token-1" | |
REMOTE_MANAGEMENT_DISABLE: "1" | |
# https://github.com/git-lfs/git-lfs/issues/5749 | |
GIT_CLONE_PROTECTION_ACTIVE: 'false' | |
jobs: | |
linting-coverage: | |
runs-on: ubuntu-latest | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
- uses: actions/setup-node@v4 | |
with: | |
node-version: '22' | |
cache: yarn | |
- name: install typescript | |
shell: bash | |
run: yarn global add typescript@4.9.5 | |
- name: install dependencies | |
run: yarn install --frozen-lockfile --network-concurrency 1 | |
- uses: actions/setup-python@v5 | |
with: | |
python-version: '3.9' | |
- uses: actions/cache@v4 | |
with: | |
path: ~/.cache/pip | |
key: ${{ runner.os }}-pip | |
- name: Install python deps | |
run: pip install flake8 | |
- name: Lint Javascript | |
run: yarn run --silent lint -- --max-warnings 0 | |
- name: Lint Markdown | |
run: yarn run --silent lint_md | |
- name: Lint python | |
run: flake8 $(git ls-files "*.py") | |
- name: Lint Yaml | |
run: yamllint -c yamllint.yml $(git ls-files "*.yml") | |
- name: Unit Coverage | |
run: | | |
set -ex | |
mkdir -p $CIRCLE_TEST_REPORTS/unit | |
yarn test | |
yarn run test_legacy_location | |
env: | |
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json | |
CIRCLE_TEST_REPORTS: /tmp | |
CIRCLE_ARTIFACTS: /tmp | |
CI_REPORTS: /tmp | |
- name: Unit Coverage logs | |
run: find /tmp/unit -exec cat {} \; | |
- name: preparing junit files for upload | |
run: | | |
mkdir -p artifacts/junit | |
find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";" | |
if: always() | |
- name: Upload files to artifacts | |
uses: scality/action-artifacts@v4 | |
with: | |
method: upload | |
url: https://artifacts.scality.net | |
user: ${{ secrets.ARTIFACTS_USER }} | |
password: ${{ secrets.ARTIFACTS_PASSWORD }} | |
source: artifacts | |
if: always() | |
build: | |
runs-on: ubuntu-latest | |
permissions: | |
contents: read | |
packages: write | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v3 | |
- name: Login to GitHub Registry | |
uses: docker/login-action@v3 | |
with: | |
registry: ghcr.io | |
username: ${{ github.repository_owner }} | |
password: ${{ github.token }} | |
- name: Build and push cloudserver image | |
uses: docker/build-push-action@v5 | |
with: | |
push: true | |
context: . | |
provenance: false | |
tags: | | |
ghcr.io/${{ github.repository }}:${{ github.sha }} | |
labels: | | |
git.repository=${{ github.repository }} | |
git.commit-sha=${{ github.sha }} | |
cache-from: type=gha,scope=cloudserver | |
cache-to: type=gha,mode=max,scope=cloudserver | |
- name: Build and push pykmip image | |
uses: docker/build-push-action@v5 | |
with: | |
push: true | |
context: .github/pykmip | |
tags: | | |
ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }} | |
labels: | | |
git.repository=${{ github.repository }} | |
git.commit-sha=${{ github.sha }} | |
cache-from: type=gha,scope=pykmip | |
cache-to: type=gha,mode=max,scope=pykmip | |
- name: Build and push MongoDB | |
uses: docker/build-push-action@v5 | |
with: | |
push: true | |
context: .github/docker/mongodb | |
tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} | |
cache-from: type=gha,scope=mongodb | |
cache-to: type=gha,mode=max,scope=mongodb | |
multiple-backend: | |
runs-on: ubuntu-latest | |
needs: build | |
env: | |
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} | |
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} | |
S3BACKEND: mem | |
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json | |
S3DATA: multiple | |
JOB_NAME: ${{ github.job }} | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
- name: Login to Registry | |
uses: docker/login-action@v3 | |
with: | |
registry: ghcr.io | |
username: ${{ github.repository_owner }} | |
password: ${{ github.token }} | |
- name: Setup CI environment | |
uses: ./.github/actions/setup-ci | |
- name: Setup CI services | |
run: docker compose --profile sproxyd up -d | |
working-directory: .github/docker | |
- name: Run multiple backend test | |
run: |- | |
set -o pipefail; | |
bash wait_for_local_port.bash 8000 40 | |
bash wait_for_local_port.bash 81 40 | |
yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/tests.log | |
env: | |
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json | |
- name: Upload logs to artifacts | |
uses: scality/action-artifacts@v4 | |
with: | |
method: upload | |
url: https://artifacts.scality.net | |
user: ${{ secrets.ARTIFACTS_USER }} | |
password: ${{ secrets.ARTIFACTS_PASSWORD }} | |
source: /tmp/artifacts | |
if: always() | |
mongo-v0-ft-tests: | |
runs-on: ubuntu-latest | |
needs: build | |
env: | |
S3BACKEND: mem | |
MPU_TESTING: "yes" | |
S3METADATA: mongodb | |
S3KMS: file | |
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json | |
DEFAULT_BUCKET_KEY_FORMAT: v0 | |
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} | |
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} | |
JOB_NAME: ${{ github.job }} | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
- name: Setup CI environment | |
uses: ./.github/actions/setup-ci | |
- name: Setup CI services | |
run: docker compose --profile mongo up -d | |
working-directory: .github/docker | |
- name: Run functional tests | |
run: |- | |
set -o pipefail; | |
bash wait_for_local_port.bash 8000 40 | |
yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log | |
env: | |
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json | |
- name: Upload logs to artifacts | |
uses: scality/action-artifacts@v4 | |
with: | |
method: upload | |
url: https://artifacts.scality.net | |
user: ${{ secrets.ARTIFACTS_USER }} | |
password: ${{ secrets.ARTIFACTS_PASSWORD }} | |
source: /tmp/artifacts | |
if: always() | |
mongo-v1-ft-tests: | |
runs-on: ubuntu-latest | |
needs: build | |
env: | |
S3BACKEND: mem | |
MPU_TESTING: "yes" | |
S3METADATA: mongodb | |
S3KMS: file | |
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json | |
DEFAULT_BUCKET_KEY_FORMAT: v1 | |
METADATA_MAX_CACHED_BUCKETS: 1 | |
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} | |
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} | |
JOB_NAME: ${{ github.job }} | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
- name: Setup CI environment | |
uses: ./.github/actions/setup-ci | |
- name: Setup CI services | |
run: docker compose --profile mongo up -d | |
working-directory: .github/docker | |
- name: Run functional tests | |
run: |- | |
set -o pipefail; | |
bash wait_for_local_port.bash 8000 40 | |
yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log | |
yarn run ft_mixed_bucket_format_version | tee /tmp/artifacts/${{ github.job }}/mixed-tests.log | |
env: | |
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json | |
- name: Upload logs to artifacts | |
uses: scality/action-artifacts@v4 | |
with: | |
method: upload | |
url: https://artifacts.scality.net | |
user: ${{ secrets.ARTIFACTS_USER }} | |
password: ${{ secrets.ARTIFACTS_PASSWORD }} | |
source: /tmp/artifacts | |
if: always() | |
file-ft-tests: | |
strategy: | |
matrix: | |
include: | |
- job-name: file-ft-tests | |
name: ${{ matrix.job-name }} | |
runs-on: ubuntu-latest | |
needs: build | |
env: | |
S3BACKEND: file | |
S3VAULT: mem | |
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} | |
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} | |
MPU_TESTING: "yes" | |
JOB_NAME: ${{ matrix.job-name }} | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
- name: Setup CI environment | |
uses: ./.github/actions/setup-ci | |
- name: Setup matrix job artifacts directory | |
shell: bash | |
run: | | |
set -exu | |
mkdir -p /tmp/artifacts/${{ matrix.job-name }}/ | |
- name: Setup CI services | |
run: docker compose up -d | |
working-directory: .github/docker | |
- name: Run file ft tests | |
run: |- | |
set -o pipefail; | |
bash wait_for_local_port.bash 8000 40 | |
yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log | |
- name: Upload logs to artifacts | |
uses: scality/action-artifacts@v4 | |
with: | |
method: upload | |
url: https://artifacts.scality.net | |
user: ${{ secrets.ARTIFACTS_USER }} | |
password: ${{ secrets.ARTIFACTS_PASSWORD }} | |
source: /tmp/artifacts | |
if: always() | |
# utapi-v2-tests: | |
# runs-on: ubuntu-latest | |
# needs: build | |
# env: | |
# ENABLE_UTAPI_V2: t | |
# S3BACKEND: mem | |
# BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket | |
# CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} | |
# MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} | |
# JOB_NAME: ${{ github.job }} | |
# steps: | |
# - name: Checkout | |
# uses: actions/checkout@v4 | |
# - name: Setup CI environment | |
# uses: ./.github/actions/setup-ci | |
# - name: Setup CI services | |
# run: docker compose up -d | |
# working-directory: .github/docker | |
# - name: Run file utapi v2 tests | |
# run: |- | |
# set -ex -o pipefail; | |
# bash wait_for_local_port.bash 8000 40 | |
# yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log | |
# - name: Upload logs to artifacts | |
# uses: scality/action-artifacts@v4 | |
# with: | |
# method: upload | |
# url: https://artifacts.scality.net | |
# user: ${{ secrets.ARTIFACTS_USER }} | |
# password: ${{ secrets.ARTIFACTS_PASSWORD }} | |
# source: /tmp/artifacts | |
# if: always() | |
quota-tests: | |
runs-on: ubuntu-latest | |
needs: build | |
strategy: | |
matrix: | |
inflights: | |
- name: "With Inflights" | |
value: "true" | |
- name: "Without Inflights" | |
value: "false" | |
env: | |
S3METADATA: mongodb | |
S3BACKEND: mem | |
S3QUOTA: scuba | |
QUOTA_ENABLE_INFLIGHTS: ${{ matrix.inflights.value }} | |
SCUBA_HOST: localhost | |
SCUBA_PORT: 8100 | |
SCUBA_HEALTHCHECK_FREQUENCY: 100 | |
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} | |
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} | |
JOB_NAME: ${{ github.job }} | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
- name: Setup CI environment | |
uses: ./.github/actions/setup-ci | |
- name: Setup CI services | |
run: docker compose --profile mongo up -d | |
working-directory: .github/docker | |
- name: Run quota tests | |
run: |- | |
set -ex -o pipefail; | |
bash wait_for_local_port.bash 8000 40 | |
yarn run test_quota | tee /tmp/artifacts/${{ github.job }}/tests.log | |
- name: Upload logs to artifacts | |
uses: scality/action-artifacts@v4 | |
with: | |
method: upload | |
url: https://artifacts.scality.net | |
user: ${{ secrets.ARTIFACTS_USER }} | |
password: ${{ secrets.ARTIFACTS_PASSWORD }} | |
source: /tmp/artifacts | |
if: always() | |
kmip-ft-tests: | |
runs-on: ubuntu-latest | |
needs: build | |
env: | |
S3BACKEND: file | |
S3VAULT: mem | |
MPU_TESTING: "yes" | |
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} | |
PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }} | |
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} | |
JOB_NAME: ${{ github.job }} | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
- name: Setup CI environment | |
uses: ./.github/actions/setup-ci | |
- name: Copy KMIP certs | |
run: cp -r ./certs /tmp/ssl-kmip | |
working-directory: .github/pykmip | |
- name: Setup CI services | |
run: docker compose --profile pykmip up -d | |
working-directory: .github/docker | |
- name: Run file KMIP tests | |
run: |- | |
set -ex -o pipefail; | |
bash wait_for_local_port.bash 8000 40 | |
bash wait_for_local_port.bash 5696 40 | |
yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log | |
- name: Upload logs to artifacts | |
uses: scality/action-artifacts@v4 | |
with: | |
method: upload | |
url: https://artifacts.scality.net | |
user: ${{ secrets.ARTIFACTS_USER }} | |
password: ${{ secrets.ARTIFACTS_PASSWORD }} | |
source: /tmp/artifacts | |
if: always() | |
ceph-backend-test: | |
runs-on: ubuntu-latest | |
needs: build | |
env: | |
S3BACKEND: mem | |
S3DATA: multiple | |
S3KMS: file | |
CI_CEPH: 'true' | |
MPU_TESTING: "yes" | |
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigCeph.json | |
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} | |
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} | |
JOB_NAME: ${{ github.job }} | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
- name: Login to GitHub Registry | |
uses: docker/login-action@v3 | |
with: | |
registry: ghcr.io | |
username: ${{ github.repository_owner }} | |
password: ${{ github.token }} | |
- name: Setup CI environment | |
uses: ./.github/actions/setup-ci | |
- uses: ruby/setup-ruby@v1 | |
with: | |
ruby-version: '2.5.9' | |
- name: Install Ruby dependencies | |
run: | | |
gem install nokogiri:1.12.5 excon:0.109.0 fog-aws:1.3.0 json:2.7.6 mime-types:3.1 rspec:3.5 | |
- name: Install Java dependencies | |
run: | | |
sudo apt-get update && sudo apt-get install -y --fix-missing default-jdk maven | |
- name: Setup CI services | |
run: docker compose --profile ceph up -d | |
working-directory: .github/docker | |
env: | |
S3METADATA: mongodb | |
- name: Run Ceph multiple backend tests | |
run: |- | |
set -ex -o pipefail; | |
bash .github/ceph/wait_for_ceph.sh | |
bash wait_for_local_port.bash 27018 40 | |
bash wait_for_local_port.bash 8000 40 | |
yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/multibackend-tests.log | |
env: | |
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json | |
S3METADATA: mem | |
- name: Run Java tests | |
run: |- | |
set -ex -o pipefail; | |
mvn test | tee /tmp/artifacts/${{ github.job }}/java-tests.log | |
working-directory: tests/functional/jaws | |
- name: Run Ruby tests | |
run: |- | |
set -ex -o pipefail; | |
rspec -fd --backtrace tests.rb | tee /tmp/artifacts/${{ github.job }}/ruby-tests.log | |
working-directory: tests/functional/fog | |
- name: Run Javascript AWS SDK tests | |
run: |- | |
set -ex -o pipefail; | |
yarn run ft_awssdk | tee /tmp/artifacts/${{ github.job }}/js-awssdk-tests.log; | |
yarn run ft_s3cmd | tee /tmp/artifacts/${{ github.job }}/js-s3cmd-tests.log; | |
env: | |
S3_LOCATION_FILE: tests/locationConfig/locationConfigCeph.json | |
S3BACKEND: file | |
S3VAULT: mem | |
S3METADATA: mongodb | |
- name: Upload logs to artifacts | |
uses: scality/action-artifacts@v4 | |
with: | |
method: upload | |
url: https://artifacts.scality.net | |
user: ${{ secrets.ARTIFACTS_USER }} | |
password: ${{ secrets.ARTIFACTS_PASSWORD }} | |
source: /tmp/artifacts | |
if: always() |