chore: update ver #5970
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CD | |
| on: | |
| push: | |
| branches: | |
| - master | |
| #on: | |
| # push: | |
| # branches-ignore: | |
| # - '**' # temporally ignore all | |
| jobs: | |
| update-schema: | |
| if: | | |
| !startsWith(github.event.head_commit.message, 'chore') && | |
| !startsWith(github.event.head_commit.message, 'build: hotfix') && | |
| !endsWith(github.event.head_commit.message, 'reformatted by jina-dev-bot') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2.5.0 | |
| with: | |
| # submodules: true | |
| fetch-depth: 100 | |
| - run: | | |
| truncate --size=24KB README.md > README-trunc.md | |
| - uses: peter-evans/dockerhub-description@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_DEVBOT_USER }} | |
| password: ${{ secrets.DOCKERHUB_DEVBOT_PWD }} | |
| repository: jinaai/jina | |
| readme-filepath: ./README-trunc.md | |
| - uses: actions/checkout@v2.5.0 | |
| with: | |
| repository: jina-ai/api | |
| path: schema | |
| token: ${{ secrets.JINA_DEV_BOT }} | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - uses: actions/setup-node@v2 | |
| with: | |
| node-version: '14' | |
| - run: | | |
| pip install . --no-cache-dir | |
| pip install docarray==0.21.0 # only valid for this version. I think we should remove this schema loading | |
| JINA_VERSION=$(sed -n '/^__version__/p' ./jina/__init__.py | cut -d \' -f2)-master | |
| echo "JINA_VERSION=${JINA_VERSION}" >> $GITHUB_ENV | |
| cd schema | |
| mkdir -p schemas | |
| jina export schema --schema-path schemas/"$JINA_VERSION.json" schemas/master.json schemas/master --yaml-path "$JINA_VERSION.yml" master.yml --json-path "$JINA_VERSION.json" master.json master | |
| python ../scripts/get-openapi-schemas.py | |
| npm install --prefix ~ snippet-enricher-cli | |
| ~/node_modules/.bin/snippet-enricher-cli --input=gateway.json --targets=shell_curl > gateway-with-code.json | |
| cd - | |
| - name: redoc-cli-gateway | |
| uses: seeebiii/redoc-cli-github-action@v10 | |
| with: | |
| args: 'bundle schema/gateway-with-code.json -o rest.html' | |
| - name: push-to-api-repo | |
| run: | | |
| mkdir -p schema/rest/ | |
| cp rest.html schema/rest/master.html | |
| cp schema/gateway.json schema/rest/master.json | |
| rm schema/gateway.json schema/gateway-with-code.json | |
| cd schema | |
| git config --local user.email "dev-bot@jina.ai" | |
| git config --local user.name "Jina Dev Bot" | |
| if [[ `git status --porcelain` ]]; then | |
| git pull && git add . && git commit -m "update ${{env.JINA_VERSION}} due to ${{github.event_name}} on ${{github.repository}}" && git push | |
| fi | |
| update-docker: | |
| needs: update-schema | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: benc-uk/workflow-dispatch@v1 | |
| with: | |
| workflow: Manual Docker Build | |
| inputs: '{ "release_token": "${{ env.release_token }}", "triggered_by": "CD"}' | |
| token: ${{ secrets.JINA_DEV_BOT }} | |
| env: | |
| release_token: ${{ secrets.JINA_CORE_RELEASE_TOKEN }} | |
| prep-testbed: | |
| runs-on: ubuntu-latest | |
| needs: update-schema | |
| env: | |
| release_token: ${{ secrets.JINA_CORE_RELEASE_TOKEN }} | |
| steps: | |
| - uses: actions/checkout@v2.5.0 | |
| - id: set-matrix | |
| run: | | |
| sudo apt-get install jq | |
| export value=$(bash scripts/get-all-test-paths.sh) | |
| echo "matrix=$value" >> $GITHUB_OUTPUT | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| docarray-v-two-test: | |
| needs: prep-testbed | |
| runs-on: ubuntu-latest | |
| env: | |
| JINA_RANDOM_PORT_MIN: 16384 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10"] | |
| protobuf-version: ['==3.19.6', ''] | |
| steps: | |
| - uses: actions/checkout@v2.5.0 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set up Golang | |
| uses: actions/setup-go@v2 | |
| with: | |
| go-version: 1.19.5 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install setuptools wheel | |
| pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code | |
| pip install cibuildwheel | |
| - name: Build wheels with setuptools-golang-build-manylinux-wheel | |
| run: | | |
| setuptools-golang-build-manylinux-wheels --pythons cp310-cp310 | |
| - name: Prepare environment | |
| run: | | |
| docker build -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| WHEEL_FILE=$(ls dist/*whl) | |
| pip install "$WHEEL_FILE[common,devel,test]" --no-cache-dir | |
| if [[ "${{ matrix.protobuf-version }}" == "==3.19.6" ]]; then | |
| pip install -U protobuf${{ matrix.protobuf-version }} grpcio==1.47.5 grpcio-reflection==1.47.5 grpcio-health-checking==1.47.5 | |
| else | |
| pip install -U protobuf${{ matrix.protobuf-version }} grpcio==1.65.5 grpcio-reflection==1.65.5 grpcio-health-checking==1.65.5 | |
| fi | |
| jina | |
| export JINA_LOG_LEVEL="ERROR" | |
| - name: Test | |
| id: test | |
| run: | | |
| pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/unit/serve/runtimes/test_helper.py | |
| pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_v2.py | |
| pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/deployment_http_composite | |
| pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_singleton.py | |
| pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_parameters_as_pydantic.py | |
| pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/test_streaming.py | |
| pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/csp | |
| pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2/docker | |
| echo "flag it as jina for codeoverage" | |
| echo "codecov_flag=jina" >> $GITHUB_OUTPUT | |
| timeout-minutes: 45 | |
| env: | |
| JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}" | |
| - name: Check codecov file | |
| id: check_files | |
| uses: andstor/file-existence-action@v1 | |
| with: | |
| files: "coverage.xml" | |
| - name: Upload coverage from test to Codecov | |
| uses: codecov/codecov-action@v3.1.1 | |
| if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8' | |
| with: | |
| file: coverage.xml | |
| name: ${{ matrix.test-path }}-codecov | |
| flags: ${{ steps.test.outputs.codecov_flag }} | |
| fail_ci_if_error: false | |
| stateful-docarray-v-two-test: | |
| needs: prep-testbed | |
| runs-on: ubuntu-latest | |
| env: | |
| JINA_RANDOM_PORT_MIN: 16384 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10"] | |
| protobuf-version: ['==3.19.6', ''] | |
| steps: | |
| - uses: actions/checkout@v2.5.0 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set up Golang | |
| uses: actions/setup-go@v2 | |
| with: | |
| go-version: 1.19.5 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install setuptools wheel | |
| pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code | |
| pip install cibuildwheel | |
| - name: Build wheels with setuptools-golang-build-manylinux-wheel | |
| run: | | |
| setuptools-golang-build-manylinux-wheels --pythons cp310-cp310 | |
| - name: Prepare environment | |
| run: | | |
| docker build -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| WHEEL_FILE=$(ls dist/*whl) | |
| pip install "$WHEEL_FILE[common,devel,test]" --no-cache-dir | |
| if [[ "${{ matrix.protobuf-version }}" == "==3.19.6" ]]; then | |
| pip install -U protobuf${{ matrix.protobuf-version }} grpcio==1.47.5 grpcio-reflection==1.47.5 grpcio-health-checking==1.47.5 | |
| else | |
| pip install -U protobuf${{ matrix.protobuf-version }} grpcio==1.65.5 grpcio-reflection==1.65.5 grpcio-health-checking==1.65.5 | |
| fi | |
| jina | |
| export JINA_LOG_LEVEL="ERROR" | |
| - name: Test stateful | |
| id: test_stateful | |
| run: | | |
| JINA_LOG_LEVEL=DEBUG JINA_RANDOM_PORT_MAX="60535" pytest --suppress-no-test-exit-code --maxfail 1 --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/stateful | |
| echo "flag it as jina for codeoverage" | |
| echo "codecov_flag=jina" >> $GITHUB_OUTPUT | |
| timeout-minutes: 30 | |
| env: | |
| JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}" | |
| - name: Check codecov file | |
| id: check_files | |
| uses: andstor/file-existence-action@v1 | |
| with: | |
| files: "coverage.xml" | |
| - name: Upload coverage from test to Codecov | |
| uses: codecov/codecov-action@v3.1.1 | |
| if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8' | |
| with: | |
| file: coverage.xml | |
| name: ${{ matrix.test-path }}-codecov | |
| flags: ${{ steps.test.outputs.codecov_flag }} | |
| fail_ci_if_error: false | |
| mark-dev-N: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 200 | |
| - name: Set up Python ${{ matrix.python }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - id: computedevdelta | |
| run: | | |
| git fetch --depth=200 | |
| LAST_VER=$(git tag -l | sort -V | tail -n1) | |
| COMMITS_SINCE_LAST_VER=$(git rev-list $LAST_VER..HEAD --count) | |
| echo "devdelta=$COMMITS_SINCE_LAST_VER" >> $GITHUB_OUTPUT | |
| INIT_FILE='jina/__init__.py' | |
| RELEASE_VER=$(sed -n '/^__version__/p' $INIT_FILE | cut -d \' -f2) | |
| echo "LAST_VER $LAST_VER" | |
| echo "COMMITS_SINCE_LAST_VER $COMMITS_SINCE_LAST_VER" | |
| echo "RELEASE_VER $RELEASE_VER" | |
| echo "devdelta=$COMMITS_SINCE_LAST_VER" >> $GITHUB_OUTPUT | |
| echo "releasever=$RELEASE_VER" >> $GITHUB_OUTPUT | |
| outputs: | |
| devdelta: ${{ steps.computedevdelta.outputs.devdelta }} | |
| releasever: ${{ steps.computedevdelta.outputs.releasever }} | |
| # Build the wheels for Linux and macOS for Python 3.8 and newer | |
| build-wheels: | |
| runs-on: ${{ matrix.os }} | |
| needs: [mark-dev-N] | |
| strategy: | |
| # Ensure that a wheel builder finishes even if another fails | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # linux | |
| - os: ubuntu-latest | |
| python: '3.8' | |
| python-manylinux-tag: "cp38-cp38" | |
| - os: ubuntu-latest | |
| python: '3.9' | |
| python-manylinux-tag: "cp39-cp39" | |
| - os: ubuntu-latest | |
| python: '3.10' | |
| python-manylinux-tag: "cp310-cp310" | |
| - os: ubuntu-latest | |
| python: '3.11' | |
| python-manylinux-tag: "cp311-cp311" | |
| # MacOS emulated | |
| - os: macos-latest | |
| python: '3.8' | |
| python-cibuildwheels: '38' | |
| platform_id: macosx_x86_64 | |
| arch: x86_64 | |
| goarch: amd64 | |
| - os: macos-latest | |
| python: '3.9' | |
| python-cibuildwheels: '39' | |
| platform_id: macosx_x86_64 | |
| arch: x86_64 | |
| goarch: amd64 | |
| - os: macos-latest | |
| python: '3.10' | |
| python-cibuildwheels: '310' | |
| platform_id: macosx_x86_64 | |
| arch: x86_64 | |
| goarch: amd64 | |
| - os: macos-latest | |
| python: '3.11' | |
| python-cibuildwheels: '311' | |
| platform_id: macosx_x86_64 | |
| arch: x86_64 | |
| goarch: amd64 | |
| # MacOS native | |
| - os: macos-latest | |
| python: '3.8' | |
| python-cibuildwheels: '38' | |
| platform_id: macosx_arm64 | |
| arch: arm64 | |
| goarch: arm64 | |
| - os: macos-latest | |
| python: '3.9' | |
| python-cibuildwheels: '39' | |
| platform_id: macosx_arm64 | |
| arch: arm64 | |
| goarch: arm64 | |
| - os: macos-latest | |
| python: '3.10' | |
| python-cibuildwheels: '310' | |
| platform_id: macosx_arm64 | |
| arch: arm64 | |
| goarch: arm64 | |
| - os: macos-latest | |
| python: '3.11' | |
| python-cibuildwheels: '311' | |
| platform_id: macosx_arm64 | |
| arch: arm64 | |
| goarch: arm64 | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v2 | |
| - name: Set up Python ${{ matrix.python }} | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Set up Golang | |
| uses: actions/setup-go@v2 | |
| with: | |
| go-version: 1.19.5 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install setuptools wheel | |
| pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code | |
| pip install cibuildwheel | |
| - name: Update version | |
| run: | | |
| DEVDELTA=${{needs.mark-dev-N.outputs.devdelta}} | |
| RELEASE_VER=${{needs.mark-dev-N.outputs.releasever}} | |
| INIT_FILE='jina/__init__.py' | |
| DEV_VER=$RELEASE_VER".dev"$DEVDELTA | |
| echo "DEVDELTA $DEVDELTA" | |
| echo "RELEASE_VER $RELEASE_VER" | |
| echo "DEV_VER $DEV_VER" | |
| if [[ "${{ matrix.os }}" == "macos-latest" ]]; then | |
| sed -i '' '/'"${RELEASE_VER}"'/s/.*/'"__version__ = '${DEV_VER}'"'/' "${INIT_FILE}" | |
| else | |
| sed -i '/'"${RELEASE_VER}"'/s/.*/'"__version__ = '${DEV_VER}'"'/' "${INIT_FILE}" | |
| fi | |
| - name: Build wheels with setuptools-golang-build-manylinux-wheel | |
| if: ${{ matrix.os == 'ubuntu-latest' }} | |
| run: | | |
| setuptools-golang-build-manylinux-wheels --pythons ${{ matrix.python-manylinux-tag }} | |
| - name: Build wheels with cibuildwheels on macos | |
| env: | |
| CIBW_BUILD: cp${{ matrix.python-cibuildwheels }}-${{ matrix.platform_id }} | |
| CIBW_ARCHS: ${{ matrix.arch }} | |
| CIBW_TEST_COMMAND: python -c "import jina; import jraft" | |
| CIBW_BUILD_VERBOSITY: 1 | |
| GOARCH: ${{ matrix.goarch }} | |
| CGO_ENABLED: 1 | |
| if: ${{ matrix.os == 'macos-latest' }} | |
| run: | | |
| python -m cibuildwheel --output-dir dist | |
| - name: Upload wheels as artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| path: dist/*.whl | |
| # comment for now, do it manually if needed | |
| # pre-release: | |
| # if: | | |
| # !startsWith(github.event.head_commit.message, 'chore') && | |
| # !startsWith(github.event.head_commit.message, 'build: hotfix') && | |
| # !endsWith(github.event.head_commit.message, 'reformatted by jina-dev-bot') | |
| # needs: [build-wheels, mark-dev-N] | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - uses: actions/checkout@v2.5.0 | |
| # with: | |
| # # submodules: true | |
| # fetch-depth: 100 | |
| # - run: | | |
| # truncate --size=24KB README.md > README-trunc.md | |
| # - uses: actions/download-artifact@v3 | |
| # with: | |
| # name: artifact | |
| # path: dist | |
| # - name: Update version | |
| # run: | | |
| # DEVDELTA=${{needs.mark-dev-N.outputs.devdelta}} | |
| # RELEASE_VER=${{needs.mark-dev-N.outputs.releasever}} | |
| # INIT_FILE='jina/__init__.py' | |
| # DEV_VER=$RELEASE_VER".dev"$DEVDELTA | |
| # echo "DEVDELTA $DEVDELTA" | |
| # echo "RELEASE_VER $RELEASE_VER" | |
| # echo "DEV_VER $DEV_VER" | |
| # sed -i '/'"${RELEASE_VER}"'/s/.*/'"__version__ = '${DEV_VER}'"'/' "${INIT_FILE}" | |
| # - name: Pre-release (.devN) | |
| # run: | | |
| # pip install twine wheel | |
| # ./scripts/release.sh | |
| # env: | |
| # TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} | |
| # TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} | |
| # JINA_SLACK_WEBHOOK: ${{ secrets.JINA_SLACK_WEBHOOK }} | |
| core-test: | |
| needs: prep-testbed | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10"] | |
| test-path: ${{fromJson(needs.prep-testbed.outputs.matrix)}} | |
| steps: | |
| - uses: actions/checkout@v2.5.0 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set up Golang | |
| uses: actions/setup-go@v2 | |
| with: | |
| go-version: 1.19.5 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install setuptools wheel | |
| pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code | |
| pip install cibuildwheel | |
| - name: Build wheels with setuptools-golang-build-manylinux-wheel | |
| run: | | |
| setuptools-golang-build-manylinux-wheels --pythons cp310-cp310 | |
| - name: Prepare environment | |
| run: | | |
| docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| WHEEL_FILE=$(ls dist/*whl) | |
| pip install "$WHEEL_FILE[all]" --no-cache-dir | |
| pip install docarray==0.21.0 | |
| pip install grpcio==1.65.5 grpcio-reflection==1.65.5 grpcio-health-checking==1.65.5 | |
| jina | |
| export JINA_LOG_LEVEL="ERROR" | |
| - name: Test | |
| id: test | |
| run: | | |
| pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/docarray_v2/*' --ignore-glob='tests/integration/stateful/*' --ignore-glob='tests/integration/hub_usage/dummyhub*' ${{ matrix.test-path }} | |
| echo "flag it as jina for codeoverage" | |
| echo "codecov_flag=jina" >> $GITHUB_OUTPUT | |
| timeout-minutes: 30 | |
| env: | |
| JINAHUB_USERNAME: ${{ secrets.JINAHUB_USERNAME }} | |
| JINAHUB_PASSWORD: ${{ secrets.JINAHUB_PASSWORD }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}" | |
| - name: Check file existence | |
| id: check_files | |
| uses: andstor/file-existence-action@v1 | |
| with: | |
| files: "coverage.xml" | |
| - name: Upload coverage from test to Codecov | |
| uses: codecov/codecov-action@v3.1.1 | |
| if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.10' | |
| with: | |
| file: coverage.xml | |
| name: ${{ matrix.test-path }}-codecov | |
| flags: ${{ steps.test.outputs.codecov_flag }} | |
| fail_ci_if_error: false | |
| import-test: | |
| runs-on: ubuntu-latest | |
| needs: update-schema | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| core: ['', 'true'] | |
| perf: ['', 'true'] | |
| python-env: ['3.7', '3.8', '3.9', '3.10', '3.11'] | |
| exclude: | |
| - core: 'true' | |
| perf: 'true' | |
| steps: | |
| - uses: actions/checkout@v2.5.0 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-env }} | |
| - name: Prepare enviroment | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| pip install --no-cache-dir . | |
| env: | |
| JINA_PIP_INSTALL_CORE: ${{ matrix.core }} | |
| JINA_PIP_INSTALL_PERF: ${{ matrix.perf }} | |
| - name: Test basic import | |
| run: python -c 'from jina import Executor,requests,Client,Flow,dynamic_batching,Document,DocumentArray' | |
| docker-image-test: | |
| needs: update-schema | |
| runs-on: ubuntu-latest | |
| services: | |
| registry: | |
| image: registry:2 | |
| ports: | |
| - 5000:5000 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test-arch: ["linux/amd64", "linux/arm64"] | |
| steps: | |
| # - name: Cancel Previous Runs | |
| # uses: styfle/cancel-workflow-action@0.9.0 | |
| # with: | |
| # access_token: ${{ github.token }} | |
| - uses: actions/checkout@v2.5.0 | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v1 | |
| with: | |
| install: true | |
| driver-opts: network=host | |
| - run: | | |
| docker run --privileged --rm tonistiigi/binfmt --uninstall qemu-aarch64 | |
| docker run --rm --privileged tonistiigi/binfmt --install all | |
| - name: Build and test | |
| uses: docker/build-push-action@v2 | |
| with: | |
| context: . | |
| file: Dockerfiles/debianx.Dockerfile | |
| platforms: ${{ matrix.test-arch }} | |
| push: true | |
| tags: localhost:5000/jina/multiarch:latest | |
| target: jina | |
| - run: | | |
| docker run --platform ${{ matrix.test-arch }} localhost:5000/jina/multiarch:latest -v | |
| hub-test: | |
| needs: update-schema | |
| runs-on: ubuntu-latest | |
| if: ${{ !github.event.pull_request.head.repo.fork }} | |
| steps: | |
| # - name: Cancel Previous Runs | |
| # uses: styfle/cancel-workflow-action@0.9.0 | |
| # with: | |
| # access_token: ${{ github.token }} | |
| - uses: actions/checkout@v2.5.0 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| - name: Test hubapp with hubpods | |
| run: | | |
| ./tests/jinahub/test_integration.sh | |
| timeout-minutes: 30 | |
| env: | |
| JINAHUB_USERNAME: ${{ secrets.JINAHUB_USERNAME }} | |
| JINAHUB_PASSWORD: ${{ secrets.JINAHUB_PASSWORD }} | |
| k8s-flow-test: | |
| needs: update-schema | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2.5.0 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| - name: Prepare enviroment | |
| run: | | |
| docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| pip install ".[all]" --no-cache-dir | |
| pip install docarray==0.21.0 | |
| jina | |
| export JINA_LOG_LEVEL="ERROR" | |
| curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl | |
| sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Test k8s Flow | |
| run: | | |
| export LINKERD2_VERSION=stable-2.11.4 | |
| curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install | sh | |
| pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_flow.py | |
| timeout-minutes: 30 | |
| - name: Check codecov file | |
| id: check_files | |
| uses: andstor/file-existence-action@v1 | |
| with: | |
| files: "coverage.xml" | |
| - name: Upload coverage from test to Codecov | |
| uses: codecov/codecov-action@v3.1.1 | |
| if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.10' | |
| with: | |
| file: coverage.xml | |
| name: ${{ matrix.test-path }}-codecov | |
| flags: ${{ steps.test.outputs.codecov_flag }} | |
| fail_ci_if_error: false | |
| k8s-deployment-test: | |
| needs: update-schema | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2.5.0 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| - name: Prepare enviroment | |
| run: | | |
| docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| pip install ".[all]" --no-cache-dir | |
| pip install docarray==0.21.0 | |
| jina | |
| export JINA_LOG_LEVEL="ERROR" | |
| curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl | |
| sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Test k8s Deployment | |
| run: | | |
| export LINKERD2_VERSION=stable-2.11.4 | |
| curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install | sh | |
| pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_deployment.py | |
| timeout-minutes: 30 | |
| - name: Check codecov file | |
| id: check_files | |
| uses: andstor/file-existence-action@v1 | |
| with: | |
| files: "coverage.xml" | |
| - name: Upload coverage from test to Codecov | |
| uses: codecov/codecov-action@v3.1.1 | |
| if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.10' | |
| with: | |
| file: coverage.xml | |
| name: ${{ matrix.test-path }}-codecov | |
| flags: ${{ steps.test.outputs.codecov_flag }} | |
| fail_ci_if_error: false | |
| k8s-graceful-test: | |
| needs: update-schema | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2.5.0 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| - name: Prepare enviroment | |
| run: | | |
| docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| pip install ".[all]" --no-cache-dir | |
| pip install docarray==0.21.0 | |
| jina | |
| export JINA_LOG_LEVEL="ERROR" | |
| curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl | |
| sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Test graceful request handling on k8s | |
| run: | | |
| export LINKERD2_VERSION=stable-2.11.4 | |
| curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install | sh | |
| pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_graceful_request_handling.py | |
| timeout-minutes: 45 | |
| - name: Check codecov file | |
| id: check_files | |
| uses: andstor/file-existence-action@v1 | |
| with: | |
| files: "coverage.xml" | |
| - name: Upload coverage from test to Codecov | |
| uses: codecov/codecov-action@v3.1.1 | |
| if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.10' | |
| with: | |
| file: coverage.xml | |
| name: ${{ matrix.test-path }}-codecov | |
| flags: ${{ steps.test.outputs.codecov_flag }} | |
| fail_ci_if_error: false | |
| k8s-failures-test: | |
| needs: update-schema | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2.5.0 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| - name: Prepare enviroment | |
| run: | | |
| docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| pip install ".[all]" --no-cache-dir | |
| pip install docarray==0.21.0 | |
| jina | |
| export JINA_LOG_LEVEL="ERROR" | |
| curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl | |
| sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Test k8s failures | |
| run: | | |
| export LINKERD2_VERSION=stable-2.11.4 | |
| curl --proto '=https' --tlsv1.2 -sSfL https://run.linkerd.io/install | sh | |
| curl --proto '=https' --tlsv1.2 -sSfL https://linkerd.github.io/linkerd-smi/install | sh | |
| pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/k8s/test_k8s_failures.py | |
| timeout-minutes: 45 | |
| - name: Check codecov file | |
| id: check_files | |
| uses: andstor/file-existence-action@v1 | |
| with: | |
| files: "coverage.xml" | |
| - name: Upload coverage from test to Codecov | |
| uses: codecov/codecov-action@v3.1.1 | |
| if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.10' | |
| with: | |
| file: coverage.xml | |
| name: ${{ matrix.test-path }}-codecov | |
| flags: ${{ steps.test.outputs.codecov_flag }} | |
| fail_ci_if_error: false | |
| k8s-otel-test: | |
| needs: update-schema | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2.5.0 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| - name: Prepare enviroment | |
| run: | | |
| docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| pip install ".[all]" --no-cache-dir | |
| pip install docarray==0.21.0 | |
| jina | |
| export JINA_LOG_LEVEL="ERROR" | |
| curl -LO https://dl.k8s.io/release/v1.21.1/bin/linux/amd64/kubectl | |
| sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Test K8s with OpenTelemetry | |
| run: | | |
| pytest -v -s --cov=jina --cov-report=xml ./tests/k8s_otel | |
| timeout-minutes: 30 | |
| - name: Check codecov file | |
| id: check_files | |
| uses: andstor/file-existence-action@v1 | |
| with: | |
| files: "coverage.xml" | |
| - name: Upload coverage from test to Codecov | |
| uses: codecov/codecov-action@v3.1.1 | |
| if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.10' | |
| with: | |
| file: coverage.xml | |
| name: ${{ matrix.test-path }}-codecov | |
| flags: ${{ steps.test.outputs.codecov_flag }} | |
| fail_ci_if_error: false | |
| docker-compose-test: | |
| needs: update-schema | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2.5.0 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| - name: Prepare enviroment | |
| run: | | |
| docker build --build-arg DOCARRAY_VERSION="0.21.0" -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip . | |
| python -m pip install --upgrade pip | |
| python -m pip install wheel | |
| pip install ".[all]" --no-cache-dir | |
| pip install docarray==0.21.0 | |
| jina | |
| export JINA_LOG_LEVEL="ERROR" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Test docker compose | |
| run: | | |
| pytest -v -s --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml ./tests/docker_compose/test_*.py | |
| timeout-minutes: 30 | |
| - name: Check codecov file | |
| id: check_files | |
| uses: andstor/file-existence-action@v1 | |
| with: | |
| files: "coverage.xml" | |
| - name: Upload coverage from test to Codecov | |
| uses: codecov/codecov-action@v3.1.1 | |
| if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.10' | |
| with: | |
| file: coverage.xml | |
| name: ${{ matrix.test-path }}-codecov | |
| flags: ${{ steps.test.outputs.codecov_flag }} | |
| fail_ci_if_error: false | |
| benchmark-pre-release: | |
| runs-on: ubuntu-latest | |
| needs: [update-schema, mark-dev-N] | |
| steps: | |
| - uses: actions/checkout@v2.5.0 | |
| with: | |
| fetch-depth: 200 | |
| - run: | | |
| DEVDELTA=${{needs.mark-dev-N.outputs.devdelta}} | |
| RELEASE_VER=${{needs.mark-dev-N.outputs.releasever}} | |
| NEXT_VER=$RELEASE_VER".dev"$COMMITS_SINCE_LAST_VER | |
| echo "NEXT_VER=$NEXT_VER" >> $GITHUB_ENV | |
| - uses: benc-uk/workflow-dispatch@v1 | |
| with: | |
| token: ${{ secrets.JINA_DEV_BOT }} | |
| workflow: Benchmark Jina | |
| repo: jina-ai/jina-terraform | |
| ref: "main" | |
| inputs: '{ "pypi_releases": "[\"${{ env.NEXT_VER }}\"]"}' | |
| # just for blocking the merge until all parallel core-test are successful | |
| success-all-steps: | |
| runs-on: ubuntu-latest | |
| needs: [core-test, docarray-v-two-test, stateful-docarray-v-two-test, import-test, hub-test, k8s-flow-test, k8s-deployment-test, k8s-graceful-test, k8s-failures-test, k8s-otel-test, docker-compose-test, docker-image-test, benchmark-pre-release, update-schema, update-docker] #, pre-release] | |
| if: always() | |
| steps: | |
| - uses: technote-space/workflow-conclusion-action@v2 | |
| - name: Check Failure | |
| if: env.WORKFLOW_CONCLUSION == 'failure' | |
| run: exit 1 | |
| - name: Success | |
| if: ${{ success() }} | |
| run: echo "All Done" |