diff --git a/.github/workflows/cleanup/action.yml b/.github/workflows/cleanup/action.yml index 16c6f6a0..33acaf5f 100644 --- a/.github/workflows/cleanup/action.yml +++ b/.github/workflows/cleanup/action.yml @@ -20,14 +20,14 @@ inputs: runs: using: "composite" steps: -# - name: Cleanup BQ Tables -# shell: bash -# run: bq rm -r -f -d ${{inputs.project_id}}:${{inputs.dataset}} + - name: Cleanup BQ Tables + shell: bash + run: bq rm -r -f -d ${{inputs.project_id}}:${{inputs.dataset}} -# - name: Cleanup GCS files -# if: always() && inputs.job_type == 'streaming' -# shell: bash -# run: gcloud storage rm gs://${{inputs.input_gcs_bucket}}/* + - name: Cleanup GCS files + if: always() && inputs.job_type == 'streaming' + shell: bash + run: gcloud storage rm gs://${{inputs.input_gcs_bucket}}/* - name: Delete pub/sub notification config if: always() && inputs.job_type == 'streaming' diff --git a/.github/workflows/configs/tests_config.json b/.github/workflows/configs/tests_config.json index 18787bf7..93e09e0e 100644 --- a/.github/workflows/configs/tests_config.json +++ b/.github/workflows/configs/tests_config.json @@ -8,13 +8,64 @@ "deid_template": "projects/dlp-dataflow-load-test/deidentifyTemplates/dlp-demo-deid-latest-1706594483019", "raw_file_pattern": "gs://input_dlp_load_test_2/largecsv100MB.csv" }, + { + "name": "Streaming2_csv", + "type": "streaming", + "file_type": "CSV", + "file_size": "500MB/min", + "gcs_file_path": "gs://input_load_test_streaming_job/*.csv", + "deid_template": "projects/dlp-dataflow-load-test/deidentifyTemplates/dlp-demo-deid-latest-1706594483019", + "raw_file_pattern": "gs://input_dlp_load_test_2/largecsv500MB.csv" + }, { "name": "Batch1_csv", "type": "batch", "file_type": "CSV", - "file_size": "100MB", - "gcs_file_path": "gs://input_dlp_load_test_2/largecsv100MB.csv", + "file_size": "500MB", + "gcs_file_path": "gs://input_dlp_load_test_2/largecsv500MB.csv", + "deid_template": "projects/dlp-dataflow-load-test/deidentifyTemplates/dlp-demo-deid-latest-1706594483019" + + }, + { + "name": "Batch2_csv", + "type": "batch", + "file_type": "CSV", + "file_size": "1GB", + "gcs_file_path": "gs://input_dlp_load_test_2/largecsv1GB.csv", + "deid_template": "projects/dlp-dataflow-load-test/deidentifyTemplates/dlp-demo-deid-latest-1706594483019" + + }, + { + "name": "Batch3_csv", + "type": "batch", + "file_type": "CSV", + "file_size": "2GB", + "gcs_file_path": "gs://input_dlp_load_test_2/largecsv2GB.csv", + "deid_template": "projects/dlp-dataflow-load-test/deidentifyTemplates/dlp-demo-deid-latest-1706594483019" + }, + { + "name": "Batch1_avro", + "type": "batch", + "file_type": "AVRO", + "file_size": "500MB", + "gcs_file_path": "gs://input_dlp_load_test_2/largeavro500MB.avro", "deid_template": "projects/dlp-dataflow-load-test/deidentifyTemplates/dlp-demo-deid-latest-1706594483019" + }, + { + "name": "Batch2_avro", + "type": "batch", + "file_type": "AVRO", + "file_size": "750MB", + "gcs_file_path": "gs://input_dlp_load_test_2/largeavro750MB.avro", + "deid_template": "projects/dlp-dataflow-load-test/deidentifyTemplates/dlp-demo-deid-latest-1706594483019" + }, + { + "name": "Batch3_avro", + "type": "batch", + "file_type": "AVRO", + "file_size": "1500MB", + "gcs_file_path": "gs://input_dlp_load_test_2/largeavro1500MB.avro", + "deid_template": "projects/dlp-dataflow-load-test/deidentifyTemplates/dlp-demo-deid-latest-1706594483019" } ] \ No newline at end of file