Skip to content

[Improvement][Alert] Add timeout params for HTTP plugin (#15174) #14391

[Improvement][Alert] Add timeout params for HTTP plugin (#15174)

[Improvement][Alert] Add timeout params for HTTP plugin (#15174) #14391

Workflow file for this run

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: Backend
on:
push:
branches:
- dev
paths:
- '.github/workflows/backend.yml'
- 'package.xml'
- 'pom.xml'
- 'dolphinscheduler-alert/**'
- 'dolphinscheduler-api/**'
- 'dolphinscheduler-common/**'
- 'dolphinscheduler-dao/**'
- 'dolphinscheduler-rpc/**'
pull_request:
concurrency:
group: backend-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
paths-filter:
name: Backend-Path-Filter
runs-on: ubuntu-latest
outputs:
not-ignore: ${{ steps.filter.outputs.not-ignore }}
db-schema: ${{ steps.filter.outputs.db-schema }}
steps:
- uses: actions/checkout@v2
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
id: filter
with:
filters: |
not-ignore:
- '!(docs/**)'
db-schema:
- 'dolphinscheduler-dao/src/main/resources/sql/**'
build:
name: Backend-Build
needs: paths-filter
if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }}
runs-on: ubuntu-latest
strategy:
matrix:
java: [ '8', '11' ]
timeout-minutes: 30
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Set up JDK ${{ matrix.java }}
uses: actions/setup-java@v2
with:
java-version: ${{ matrix.java }}
distribution: 'adopt'
- name: Sanity Check
uses: ./.github/actions/sanity-check
with:
token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/cache@v3
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
restore-keys: ${{ runner.os }}-maven-
- name: Build and Package on ${{ matrix.java }}
run: |
./mvnw -B clean install \
-Prelease \
-Dmaven.test.skip=true \
-Dspotless.skip=true \
-Dhttp.keepAlive=false \
-Dmaven.wagon.http.pool=false \
-Dmaven.wagon.httpconnectionManager.ttlSeconds=120
- name: Check dependency license
run: tools/dependencies/check-LICENSE.sh
- uses: actions/upload-artifact@v2
if: ${{ matrix.java == '8' }}
name: Upload Binary Package
with:
name: binary-package-${{ matrix.java }}
path: ./dolphinscheduler-dist/target/apache-dolphinscheduler-*-SNAPSHOT-bin.tar.gz
retention-days: 1
cluster-test:
name: ${{ matrix.case.name }}
needs: build
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
matrix:
case:
- name: cluster-test-mysql
script: .github/workflows/cluster-test/mysql/start-job.sh
- name: cluster-test-postgresql
script: .github/workflows/cluster-test/postgresql/start-job.sh
steps:
- uses: actions/checkout@v2
with:
submodules: true
- uses: actions/download-artifact@v2
name: Download Binary Package
with:
# Only run cluster test on jdk8
name: binary-package-8
path: ./
- name: Running cluster test
run: |
/bin/bash ${{ matrix.case.script }}
schema-check:
runs-on: ubuntu-latest
if: ${{ (needs.paths-filter.outputs.db-schema == 'true') || (github.event_name == 'push') }}
timeout-minutes: 20
needs: build
services:
mysql:
image: mysql:5.7
env:
MYSQL_ROOT_PASSWORD: mysql
MYSQL_DATABASE: dolphinscheduler_dev
ports:
- 3306:3306
options: --name=mysql --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
postgres:
image: postgres:15
env:
POSTGRES_PASSWORD: postgres
POSTGRES_DB: dolphinscheduler_dev
ports:
- 5432:5432
options: --name=postgres --health-cmd=pg_isready --health-interval=10s --health-timeout=5s --health-retries=5
strategy:
fail-fast: false
matrix:
db: ["mysql", "postgresql"]
version: ["2.0.9", "3.0.6", "3.1.8"]
steps:
- name: Set up JDK 8
uses: actions/setup-java@v2
with:
java-version: 8
distribution: 'adopt'
- name: Install Atlas and Create Dir
run: |
mkdir -p dolphinscheduler/dev dolphinscheduler/${{ matrix.version }}
curl -sSf https://atlasgo.sh | sh
- name: Download Tarball
uses: actions/download-artifact@v2
with:
name: binary-package-8
path: dolphinscheduler/dev
- name: Set Env
run: |
VERSION=${{ matrix.version }}
echo "DATABASE_VERSION=${VERSION//\./}" >> $GITHUB_ENV
- name: Prepare
run: |
wget https://dlcdn.apache.org/dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -P dolphinscheduler/${{ matrix.version }}
tar -xzf dolphinscheduler/${{ matrix.version }}/apache-dolphinscheduler-${{ matrix.version }}-bin.tar.gz -C dolphinscheduler/${{ matrix.version }} --strip-components 1
tar -xzf dolphinscheduler/dev/apache-dolphinscheduler-*-bin.tar.gz -C dolphinscheduler/dev --strip-components 1
if [[ ${{ matrix.db }} == "mysql" ]]; then
MYSQL_JDBC_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar"
MYSQL_JDBC_JAR="mysql-connector-java-8.0.16.jar"
wget ${MYSQL_JDBC_URL} -O /tmp/${MYSQL_JDBC_JAR}
for base_dir in dolphinscheduler/dev dolphinscheduler/${{ matrix.version }}; do
if [[ $base_dir == *"dolphinscheduler/2"* ]]; then
cp /tmp/${MYSQL_JDBC_JAR} ${base_dir}/lib
else
for d in alert-server api-server master-server worker-server tools; do
cp /tmp/${MYSQL_JDBC_JAR} ${base_dir}/${d}/libs
done
fi
done
docker exec -i mysql mysql -uroot -pmysql -e "create database dolphinscheduler_${{ env.DATABASE_VERSION }}";
else
docker exec -i postgres psql -U postgres -c "create database dolphinscheduler_${{ env.DATABASE_VERSION }};"
fi
- name: Check
run: |
if [[ $DATABASE_VERSION -lt 300 ]]; then
chmod +x dolphinscheduler/dev/tools/bin/upgrade-schema.sh dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
else
chmod +x dolphinscheduler/dev/tools/bin/upgrade-schema.sh dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
fi
if [[ ${{ matrix.db }} == "mysql" ]]; then
export DATABASE="mysql"
export SPRING_DATASOURCE_DRIVER_CLASS_NAME="com.mysql.cj.jdbc.Driver"
export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_dev?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false"
export SPRING_DATASOURCE_USERNAME="root"
export SPRING_DATASOURCE_PASSWORD="mysql"
bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
export SPRING_DATASOURCE_URL="jdbc:mysql://127.0.0.1:3306/dolphinscheduler_${{ env.DATABASE_VERSION }}?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false"
if [[ $DATABASE_VERSION -lt 300 ]]; then
bash dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
else
bash dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
fi
bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
atlas_result=$(atlas schema diff \
--from "mysql://root:mysql@127.0.0.1:3306/dolphinscheduler_${{ env.DATABASE_VERSION }}" \
--to "mysql://root:mysql@127.0.0.1:3306/dolphinscheduler_dev")
if [[ ${atlas_result} != *"Schemas are synced"* ]]; then
echo "================================================================================================"
echo " !!!!! For Contributors !!!!!"
echo "================================================================================================"
echo "Database schema not sync, please add below change in the latest version of dolphinscheduler-dao/src/main/resources/sql/upgrade directory"
echo "${atlas_result}"
exit 1
fi
else
export DATABASE="postgresql"
export SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver"
export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_dev"
export SPRING_DATASOURCE_USERNAME="postgres"
export SPRING_DATASOURCE_PASSWORD="postgres"
bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
export SPRING_DATASOURCE_URL="jdbc:postgresql://127.0.0.1:5432/dolphinscheduler_${{ env.DATABASE_VERSION }}"
if [[ $DATABASE_VERSION -lt 300 ]]; then
bash dolphinscheduler/${{ matrix.version }}/script/create-dolphinscheduler.sh
else
bash dolphinscheduler/${{ matrix.version }}/tools/bin/upgrade-schema.sh
fi
bash dolphinscheduler/dev/tools/bin/upgrade-schema.sh
atlas_result=$(atlas schema diff \
--from "postgres://postgres:postgres@127.0.0.1:5432/dolphinscheduler_${{ env.DATABASE_VERSION }}?search_path=public&sslmode=disable" \
--to "postgres://postgres:postgres@127.0.0.1:5432/dolphinscheduler_dev?search_path=public&sslmode=disable")
if [[ ${atlas_result} != *"Schemas are synced"* ]]; then
echo "================================================================================================"
echo " !!!!! For Contributors !!!!!"
echo "================================================================================================"
echo "Database schema not sync, please add below change in the latest version in dolphinscheduler-dao/src/main/resources/sql/upgrade directory"
echo "${atlas_result}"
exit 1
fi
fi
result:
name: Build
runs-on: ubuntu-latest
timeout-minutes: 30
needs: [ build, paths-filter, cluster-test, schema-check ]
if: always()
steps:
- name: Status
run: |
if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ needs.paths-filter.outputs.db-schema }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
echo "Skip Build!"
exit 0
fi
if [[ ${{ needs.build.result }} != 'success' || ${{ needs.cluster-test.result }} != 'success' ]]; then
echo "Build Failed!"
exit -1
fi