test framework pipeline #4
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Load testing | |
on: | |
pull_request: | |
types: [ opened, reopened, synchronize ] | |
branches: | |
- pipeline_performance | |
- master | |
workflow_dispatch: | |
inputs: | |
test_config_json: | |
description: test configs | |
type: String | |
env: | |
PROJECT_ID: "dlp-dataflow-load-test" | |
REGION: "us-central1" | |
INSPECT_TEMPLATE: "projects/dlp-dataflow-load-test/inspectTemplates/dlp-demo-inspect-latest-1706594483019" | |
DEID_TEMPLATE: "projects/dlp-dataflow-load-test/deidentifyTemplates/dlp-demo-deid-latest-1706594483019" | |
PUB_SUB_TOPIC: "projects/dlp-dataflow-load-test/topics/load_test_pub_sub_topic" | |
CLOUD_WORKFLOW: "generate_files_workflow" | |
jobs: | |
generate-uuid: | |
runs-on: ubuntu-latest | |
timeout-minutes: 5 | |
outputs: | |
uuid: ${{ steps.gen-uuid.outputs.uuid }} | |
steps: | |
- name: Generate UUID for workflow | |
id: gen-uuid | |
run: | | |
new_uuid=$(uuidgen) | |
modified_uuid=$(echo "$new_uuid" | cut -c1-8 ) | |
echo "uuid=$modified_uuid" >> "$GITHUB_OUTPUT" | |
pre-processing: | |
needs: generate-uuid | |
runs-on: [self-hosted, load-testing] | |
timeout-minutes: 5 | |
outputs: | |
matrix: ${{ steps.set-matrix.outputs.matrix }} | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Read test details | |
id: set-matrix | |
run: | | |
matrix=$(jq -c . < .github/workflows/configs/tests_config.json) | |
if [[ "${{github.event.inputs.test_config_json}}" != "" ]]; then | |
matrix=$(echo '${{github.event.inputs.test_config_json}}' | jq .) | |
fi | |
echo "matrix={\"include\":$(echo $matrix)}" >> $GITHUB_OUTPUT | |
echo $matrix | |
# run-test: | |
# needs: | |
# - generate-uuid | |
# - pre-processing | |
# | |
# runs-on: [self-hosted, load-testing] | |
# continue-on-error: true | |
# strategy: | |
# max-parallel: 1 | |
# matrix: ${{ fromJSON(needs.pre-processing.outputs.matrix) }} | |
# | |
# steps: | |
# - name: Set job parameters | |
# id: set-job-params | |
# run: | | |
# uuid=${{needs.generate-uuid.outputs.uuid}} | |
# test_name=$(echo "${{matrix.name}}" | tr '_' '-') | |
# echo "job_name=load-test-${{needs.generate-uuid.outputs.uuid}}-test-$test_name" >> $GITHUB_OUTPUT | |
# echo "dataset=dataset_${{needs.generate-uuid.outputs.uuid}}_${{matrix.name}}" >> $GITHUB_OUTPUT | |
# echo "Test details: ${{matrix.name}}" | |
# echo "job_name=load-test-${{needs.generate-uuid.outputs.uuid}}-test-$test_name" | |
# echo "dataset=dataset_${{needs.generate-uuid.outputs.uuid}}_${{matrix.name}}" | |
# input_gcs_bucket=$(echo "${{ matrix.gcs_file_path }}" | awk -F "/" '{print $3}') | |
# echo "input_gcs_bucket=$input_gcs_bucket" >> $GITHUB_OUTPUT | |
# | |
# - name: Submit dataflow job | |
# id: submit-dataflow-job | |
# uses: ./.github/workflows/submit-dataflow-job | |
# with: | |
# project_id: ${{env.PROJECT_ID}} | |
# input_gcs_bucket: ${{ steps.set-job-params.outputs.input_gcs_bucket }} | |
# gcs_file_path: ${{ matrix.gcs_file_path }} | |
# dataset: ${{ steps.set-job-params.outputs.dataset }} | |
# inspect_template: ${{env.INSPECT_TEMPLATE}} | |
# deid_template: ${{ matrix.deid_template }} | |
# job_name: ${{steps.set-job-params.outputs.job_name}} | |
# job_type: ${{ matrix.type }} | |
# gcs_notification_topic: ${{env.PUB_SUB_TOPIC}} | |
# | |
# - name: execute copy files workflow for streaming jobs | |
# id: copy-files | |
# if: always() && matrix.type == 'streaming' | |
# uses: ./.github/workflows/execute-copy-workflow | |
# with: | |
# raw_bucket: ${{ matrix.source_file_bucket }} | |
# raw_file_pattern: ${{ matrix.raw_file_pattern }} | |
# input_gcs_bucket: ${{ steps.set-job-params.outputs.input_gcs_bucket }} | |
# job_id: ${{steps.submit-dataflow-job.outputs.job_id}} | |
# workflow_name: ${{env.CLOUD_WORKFLOW}} | |
# region: ${{env.REGION}} | |
# | |
# - name: Poll till job finishes | |
# uses: ./.github/workflows/poll-job | |
# with: | |
# job_id: ${{steps.submit-dataflow-job.outputs.job_id}} | |
# region: ${{env.REGION}} | |
# | |
# - name: Fetch metrics | |
# uses: ./.github/workflows/fetch-metrics | |
# with: | |
# job_id: ${{steps.submit-dataflow-job.outputs.job_id}} | |
# project_id: ${{env.PROJECT_ID}} | |
# test_uuid: ${{needs.generate-uuid.outputs.uuid}} | |
# test_name: ${{ matrix.name }} | |
# test_details: ${{ toJSON(matrix) }} | |
# | |
# - name: Cleanup | |
# if: always() | |
# uses: ./.github/workflows/cleanup | |
# with: | |
# project_id: ${{env.PROJECT_ID}} | |
# job_id: ${{steps.submit-dataflow-job.outputs.job_id}} | |
# dataset: ${{steps.set-job-params.outputs.dataset}} | |
# input_gcs_bucket: ${{ steps.set-job-params.outputs.input_gcs_bucket }} | |
# job_type: ${{ matrix.type }} | |
# publish-test-results: | |
# needs: | |
# - generate-uuid | |
# - pre-processing | |
# - run-test | |
# runs-on: [self-hosted, load-testing] | |
# steps: | |
# - uses: actions/checkout@v2 | |
# | |
# - name: Execute publishMetrics Script | |
# run: | | |
# python3 .github/workflows/scripts/publishTestReport.py ${{env.PROJECT_ID}} ${{ needs.generate-uuid.outputs.uuid }} | |
# | |
# | |
# |