Performance testing #16
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Load testing | |
| on: | |
| pull_request: | |
| types: [ opened, reopened, synchronize ] | |
| branches: | |
| - pipeline_performance | |
| - master | |
| workflow_dispatch: | |
| inputs: | |
| test_config_json: | |
| description: test configs | |
| type: String | |
| env: | |
| PROJECT_ID: "dlp-dataflow-load-test" | |
| REGION: "us-central1" | |
| INSPECT_TEMPLATE: "projects/dlp-dataflow-load-test/inspectTemplates/dlp-demo-inspect-latest-1706594483019" | |
| DEID_TEMPLATE: "projects/dlp-dataflow-load-test/deidentifyTemplates/dlp-demo-deid-latest-1706594483019" | |
| PUB_SUB_TOPIC: "projects/dlp-dataflow-load-test/topics/load_test_pub_sub_topic" | |
| CLOUD_WORKFLOW: "generate_files_workflow" | |
| jobs: | |
| generate-uuid: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| outputs: | |
| uuid: ${{ steps.gen-uuid.outputs.uuid }} | |
| steps: | |
| - name: Generate UUID for workflow | |
| id: gen-uuid | |
| run: | | |
| new_uuid=$(uuidgen) | |
| modified_uuid=$(echo "$new_uuid" | cut -c1-8 ) | |
| echo "uuid=$modified_uuid" >> "$GITHUB_OUTPUT" | |
| pre-processing: | |
| needs: generate-uuid | |
| runs-on: [self-hosted, load-testing] | |
| timeout-minutes: 5 | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Read test details | |
| id: set-matrix | |
| run: | | |
| matrix=$(jq -c . < .github/workflows/configs/tests_config.json) | |
| if [[ "${{github.event.inputs.test_config_json}}" != "" ]]; then | |
| matrix=$(echo '${{github.event.inputs.test_config_json}}' | jq .) | |
| fi | |
| echo "matrix={\"include\":$(echo $matrix)}" >> $GITHUB_OUTPUT | |
| echo $matrix | |
| run-test: | |
| needs: | |
| - generate-uuid | |
| - pre-processing | |
| runs-on: [self-hosted, load-testing] | |
| continue-on-error: true | |
| strategy: | |
| max-parallel: 1 | |
| matrix: ${{ fromJSON(needs.pre-processing.outputs.matrix) }} | |
| steps: | |
| - name: Set job parameters | |
| id: set-job-params | |
| run: | | |
| uuid=${{needs.generate-uuid.outputs.uuid}} | |
| test_name=$(echo "${{matrix.name}}" | tr '_' '-') | |
| echo "job_name=load-test-${{needs.generate-uuid.outputs.uuid}}-test-$test_name" >> $GITHUB_OUTPUT | |
| echo "dataset=dataset_${{needs.generate-uuid.outputs.uuid}}_${{matrix.name}}" >> $GITHUB_OUTPUT | |
| echo "Test details: ${{matrix.name}}" | |
| echo "job_name=load-test-${{needs.generate-uuid.outputs.uuid}}-test-$test_name" | |
| echo "dataset=dataset_${{needs.generate-uuid.outputs.uuid}}_${{matrix.name}}" | |
| input_gcs_bucket=$(echo "${{ matrix.gcs_file_path }}" | awk -F "/" '{print $3}') | |
| echo "input_gcs_bucket=$input_gcs_bucket" >> $GITHUB_OUTPUT | |
| - name: Submit dataflow job | |
| id: submit-dataflow-job | |
| uses: ./.github/workflows/submit-dataflow-job | |
| with: | |
| project_id: ${{env.PROJECT_ID}} | |
| input_gcs_bucket: ${{ steps.set-job-params.outputs.input_gcs_bucket }} | |
| gcs_file_path: ${{ matrix.gcs_file_path }} | |
| dataset: ${{ steps.set-job-params.outputs.dataset }} | |
| inspect_template: ${{env.INSPECT_TEMPLATE}} | |
| deid_template: ${{ matrix.deid_template }} | |
| job_name: ${{steps.set-job-params.outputs.job_name}} | |
| job_type: ${{ matrix.type }} | |
| gcs_notification_topic: ${{env.PUB_SUB_TOPIC}} | |
| - name: execute copy files workflow for streaming jobs | |
| id: copy-files | |
| if: always() && matrix.type == 'streaming' | |
| uses: ./.github/workflows/execute-copy-workflow | |
| with: | |
| raw_bucket: ${{ matrix.source_file_bucket }} | |
| raw_file_pattern: ${{ matrix.raw_file_pattern }} | |
| input_gcs_bucket: ${{ steps.set-job-params.outputs.input_gcs_bucket }} | |
| job_id: ${{steps.submit-dataflow-job.outputs.job_id}} | |
| workflow_name: ${{env.CLOUD_WORKFLOW}} | |
| region: ${{env.REGION}} | |
| - name: Poll till job finishes | |
| uses: ./.github/workflows/poll-job | |
| with: | |
| job_id: ${{steps.submit-dataflow-job.outputs.job_id}} | |
| region: ${{env.REGION}} | |
| - name: Fetch metrics | |
| uses: ./.github/workflows/fetch-metrics | |
| with: | |
| job_id: ${{steps.submit-dataflow-job.outputs.job_id}} | |
| project_id: ${{env.PROJECT_ID}} | |
| test_uuid: ${{needs.generate-uuid.outputs.uuid}} | |
| test_name: ${{ matrix.name }} | |
| test_details: ${{ toJSON(matrix) }} | |
| - name: Cleanup | |
| if: always() | |
| uses: ./.github/workflows/cleanup | |
| with: | |
| project_id: ${{env.PROJECT_ID}} | |
| job_id: ${{steps.submit-dataflow-job.outputs.job_id}} | |
| dataset: ${{steps.set-job-params.outputs.dataset}} | |
| input_gcs_bucket: ${{ steps.set-job-params.outputs.input_gcs_bucket }} | |
| job_type: ${{ matrix.type }} | |
| publish-test-results: | |
| needs: | |
| - generate-uuid | |
| - pre-processing | |
| - run-test | |
| runs-on: [self-hosted, load-testing] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Execute publishMetrics Script | |
| run: | | |
| python3 .github/workflows/scripts/publishTestReport.py ${{env.PROJECT_ID}} ${{ needs.generate-uuid.outputs.uuid }} | |