Skip to content

wip

wip #189

Workflow file for this run

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: build-branch
on:
pull_request:
types: [opened, ready_for_review, synchronize]
push:
schedule:
- cron: 30 0,12 * * *
env:
FAIL_FAST: ${{ github.event_name == 'pull_request' }}
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
concurrency:
group: ci-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
build-info:
runs-on: ubuntu-20.04
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
outputs:
basic-checks: ${{ steps.selective-checks.outputs.basic-checks }}
needs-basic-checks: ${{ steps.selective-checks.outputs.needs-basic-checks }}
needs-build: ${{ steps.selective-checks.outputs.needs-build }}
needs-compile: ${{ steps.selective-checks.outputs.needs-compile }}
needs-compose-tests: ${{ steps.selective-checks.outputs.needs-compose-tests }}
needs-dependency-check: ${{ steps.selective-checks.outputs.needs-dependency-check }}
needs-integration-tests: ${{ steps.selective-checks.outputs.needs-integration-tests }}
needs-kubernetes-tests: ${{ steps.selective-checks.outputs.needs-kubernetes-tests }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
- name: Fetch incoming commit ${{ github.sha }} with its parent
uses: actions/checkout@v3
with:
ref: ${{ github.sha }}
fetch-depth: 2
persist-credentials: false
if: github.event_name == 'pull_request'
- name: Selective checks
id: selective-checks
env:
PR_LABELS: "${{ toJSON(github.event.pull_request.labels.*.name) }}"
PR_DRAFT: "${{ github.event.pull_request.draft }}"
run: |
if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
# Run selective checks
dev-support/ci/selective_ci_checks.sh "${GITHUB_SHA}"
else
# Run all checks
dev-support/ci/selective_ci_checks.sh
fi
build:
needs:
- build-info
runs-on: ubuntu-20.04
timeout-minutes: 30
if: needs.build-info.outputs.needs-build == 'true'
strategy:
matrix:
java: [ 8 ]
fail-fast: false
steps:
- name: Checkout project
uses: actions/checkout@v3
- name: Cache for npm dependencies
uses: actions/cache@v3
with:
path: |
~/.pnpm-store
**/node_modules
key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
restore-keys: |
${{ runner.os }}-pnpm-
- name: Cache for maven dependencies
uses: actions/cache@v3
with:
path: ~/.m2/repository
key: maven-repo-${{ hashFiles('**/pom.xml') }}-${{ matrix.java }}
restore-keys: |
maven-repo-${{ hashFiles('**/pom.xml') }}
maven-repo-
- name: Setup java
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: ${{ matrix.java }}
- name: Run a full build
run: hadoop-ozone/dev-support/checks/build.sh -Pcoverage -Pdist -Psrc
- name: Store binaries for tests
uses: actions/upload-artifact@v3
with:
name: ozone-bin
path: |
hadoop-ozone/dist/target/ozone-*.tar.gz
!hadoop-ozone/dist/target/ozone-*-src.tar.gz
retention-days: 1
- name: Store source tarball for compilation
uses: actions/upload-artifact@v3
with:
name: ozone-src
path: hadoop-ozone/dist/target/ozone-*-src.tar.gz
retention-days: 1
- name: Delete temporary build artifacts before caching
run: |
#Never cache local artifacts
rm -rf ~/.m2/repository/org/apache/ozone/hdds*
rm -rf ~/.m2/repository/org/apache/ozone/ozone*
if: always()
compile:
needs:
- build-info
- build
runs-on: ubuntu-20.04
timeout-minutes: 30
if: needs.build-info.outputs.needs-compile == 'true'
strategy:
matrix:
java: [ 11 ]
fail-fast: false
steps:
- name: Download Ozone source tarball
uses: actions/download-artifact@v3
with:
name: ozone-src
- name: Untar sources
run: |
tar --strip-components 1 -xzvf ozone*-src.tar.gz
- name: Cache for maven dependencies
uses: actions/cache@v3
with:
path: ~/.m2/repository
key: maven-repo-${{ hashFiles('**/pom.xml') }}-${{ matrix.java }}
restore-keys: |
maven-repo-${{ hashFiles('**/pom.xml') }}
maven-repo-
- name: Setup java
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: ${{ matrix.java }}
- name: Compile Ozone using Java ${{ matrix.java }}
run: hadoop-ozone/dev-support/checks/build.sh -Dskip.npx -Dskip.installnpx
- name: Delete temporary build artifacts before caching
run: |
#Never cache local artifacts
rm -rf ~/.m2/repository/org/apache/ozone/hdds*
rm -rf ~/.m2/repository/org/apache/ozone/ozone*
if: always()
basic:
needs:
- build-info
runs-on: ubuntu-20.04
timeout-minutes: 90
if: needs.build-info.outputs.needs-basic-checks == 'true'
strategy:
matrix:
check: ${{ fromJson(needs.build-info.outputs.basic-checks) }}
fail-fast: false
steps:
- name: Checkout project
uses: actions/checkout@v3
if: matrix.check != 'bats'
- name: Checkout project with history
uses: actions/checkout@v3
with:
fetch-depth: 0
if: matrix.check == 'bats'
- name: Cache for maven dependencies
uses: actions/cache@v3
with:
path: ~/.m2/repository
key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ matrix.check }}
restore-keys: |
maven-repo-${{ hashFiles('**/pom.xml') }}-8
maven-repo-${{ hashFiles('**/pom.xml') }}
maven-repo-
if: ${{ !contains('author,bats,docs', matrix.check) }}
- name: Setup java
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: 8
- name: Execute tests
run: hadoop-ozone/dev-support/checks/${{ matrix.check }}.sh
- name: Summary of failures
run: cat target/${{ matrix.check }}/summary.txt
if: ${{ !cancelled() }}
- name: Archive build results
uses: actions/upload-artifact@v3
if: ${{ !cancelled() }}
with:
name: ${{ matrix.check }}
path: target/${{ matrix.check }}
continue-on-error: true
- name: Delete temporary build artifacts before caching
run: |
#Never cache local artifacts
rm -rf ~/.m2/repository/org/apache/ozone/hdds*
rm -rf ~/.m2/repository/org/apache/ozone/ozone*
if: always()
dependency:
needs:
- build-info
- build
runs-on: ubuntu-20.04
timeout-minutes: 5
if: needs.build-info.outputs.needs-dependency-check == 'true'
steps:
- name: Checkout project
uses: actions/checkout@v3
- name: Download compiled Ozone binaries
uses: actions/download-artifact@v3
with:
name: ozone-bin
- name: Untar binaries
run: |
mkdir dist
tar -C dist --strip-components 1 -xzf ozone*.tar.gz
- name: Execute tests
run: |
export OZONE_DIST_DIR=`pwd`/dist
./hadoop-ozone/dev-support/checks/dependency.sh
- name: Archive build results
uses: actions/upload-artifact@v3
if: always()
with:
name: dependency
path: target/dependency
continue-on-error: true
acceptance:
needs:
- build-info
- build
runs-on: ubuntu-20.04
timeout-minutes: 150
if: needs.build-info.outputs.needs-compose-tests == 'true'
strategy:
matrix:
suite:
- secure
- unsecure
- compat
- HA
- MR
- misc
fail-fast: false
steps:
- name: Checkout project
uses: actions/checkout@v3
- name: Download compiled Ozone binaries
uses: actions/download-artifact@v3
with:
name: ozone-bin
- name: Untar binaries
run: |
mkdir -p hadoop-ozone/dist/target
tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target
sudo chmod -R a+rwX hadoop-ozone/dist/target
- name: Execute tests
run: |
pushd hadoop-ozone/dist/target/ozone-*
sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws
popd
./hadoop-ozone/dev-support/checks/acceptance.sh
env:
KEEP_IMAGE: false
OZONE_ACCEPTANCE_SUITE: ${{ matrix.suite }}
OZONE_WITH_COVERAGE: true
OZONE_VOLUME_OWNER: 1000
- name: Archive build results
uses: actions/upload-artifact@v3
if: always()
with:
name: acceptance-${{ matrix.suite }}
path: target/acceptance
continue-on-error: true
kubernetes:
needs:
- build-info
- build
runs-on: ubuntu-20.04
timeout-minutes: 60
if: needs.build-info.outputs.needs-kubernetes-tests == 'true'
steps:
- name: Checkout project
uses: actions/checkout@v3
- name: Download compiled Ozone binaries
uses: actions/download-artifact@v3
with:
name: ozone-bin
- name: Untar binaries
run: |
mkdir -p hadoop-ozone/dist/target
tar xzvf ozone*.tar.gz -C hadoop-ozone/dist/target
- name: Execute tests
run: |
pushd hadoop-ozone/dist/target/ozone-*
sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws
popd
./hadoop-ozone/dev-support/checks/kubernetes.sh
- name: Archive build results
uses: actions/upload-artifact@v3
if: always()
with:
name: kubernetes
path: target/kubernetes
continue-on-error: true
integration:
needs:
- build-info
runs-on: ubuntu-20.04
timeout-minutes: 150
if: needs.build-info.outputs.needs-integration-tests == 'true'
strategy:
matrix:
profile:
- client
- filesystem
- hdds
- om
- ozone
- scm
- flaky
fail-fast: false
steps:
- name: Checkout project
uses: actions/checkout@v3
- name: Cache for maven dependencies
uses: actions/cache@v3
with:
path: ~/.m2/repository
key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ matrix.profile }}
restore-keys: |
maven-repo-${{ hashFiles('**/pom.xml') }}-8
maven-repo-${{ hashFiles('**/pom.xml') }}
maven-repo-
- name: Setup java
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: 8
- name: Execute tests
run: hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }}
if: matrix.profile != 'flaky'
- name: Execute flaky tests
run: hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }} -Dsurefire.rerunFailingTestsCount=5 -Dsurefire.fork.timeout=3600
if: matrix.profile == 'flaky'
- name: Summary of failures
run: cat target/${{ github.job }}/summary.txt
if: always()
- name: Archive build results
uses: actions/upload-artifact@v3
if: always()
with:
name: it-${{ matrix.profile }}
path: target/integration
continue-on-error: true
- name: Delete temporary build artifacts before caching
run: |
#Never cache local artifacts
rm -rf ~/.m2/repository/org/apache/ozone/hdds*
rm -rf ~/.m2/repository/org/apache/ozone/ozone*
if: always()
coverage:
runs-on: ubuntu-20.04
timeout-minutes: 30
if: github.repository == 'apache/ozone' && github.event_name != 'pull_request'
needs:
- acceptance
- basic
- integration
steps:
- name: Checkout project
uses: actions/checkout@v3
- name: Cache for maven dependencies
uses: actions/cache@v3
with:
path: ~/.m2/repository
key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ github.job }}
restore-keys: |
maven-repo-${{ hashFiles('**/pom.xml') }}-8
maven-repo-${{ hashFiles('**/pom.xml') }}
maven-repo-
- name: Download artifacts
uses: actions/download-artifact@v3
with:
path: target/artifacts
- name: Untar binaries
run: |
mkdir -p hadoop-ozone/dist/target
tar xzvf target/artifacts/ozone-bin/ozone*.tar.gz -C hadoop-ozone/dist/target
- name: Calculate combined coverage
run: ./hadoop-ozone/dev-support/checks/coverage.sh
- name: Setup java 11
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: 11
- name: Upload coverage to Sonar
run: ./hadoop-ozone/dev-support/checks/sonar.sh
env:
SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Archive build results
uses: actions/upload-artifact@v3
with:
name: coverage
path: target/coverage
continue-on-error: true
- name: Delete temporary build artifacts before caching
run: |
#Never cache local artifacts
rm -rf ~/.m2/repository/org/apache/ozone/hdds*
rm -rf ~/.m2/repository/org/apache/ozone/ozone*
if: always()