minor changes

GoogleCloudPlatform · Mar 8, 2024 · 23a3357 · 23a3357
1 parent ec3274e
commit 23a3357
Show file tree

Hide file tree

Showing 2 changed files with 60 additions and 9 deletions.
diff --git a/.github/workflows/cleanup/action.yml b/.github/workflows/cleanup/action.yml
@@ -20,14 +20,14 @@ inputs:
 runs:
   using: "composite"
   steps:
-#    - name: Cleanup BQ Tables
-#      shell: bash
-#      run: bq rm -r -f -d ${{inputs.project_id}}:${{inputs.dataset}}
+    - name: Cleanup BQ Tables
+      shell: bash
+      run: bq rm -r -f -d ${{inputs.project_id}}:${{inputs.dataset}}
 
-#    - name: Cleanup GCS files
-#      if: always() && inputs.job_type == 'streaming'
-#      shell: bash
-#      run: gcloud storage rm gs://${{inputs.input_gcs_bucket}}/*
+    - name: Cleanup GCS files
+      if: always() && inputs.job_type == 'streaming'
+      shell: bash
+      run: gcloud storage rm gs://${{inputs.input_gcs_bucket}}/*
 
     - name: Delete pub/sub notification config
       if: always() && inputs.job_type == 'streaming'

diff --git a/.github/workflows/configs/tests_config.json b/.github/workflows/configs/tests_config.json
@@ -8,13 +8,64 @@
     "deid_template": "projects/dlp-dataflow-load-test/deidentifyTemplates/dlp-demo-deid-latest-1706594483019",
     "raw_file_pattern": "gs://input_dlp_load_test_2/largecsv100MB.csv"
   },
+  {
+    "name": "Streaming2_csv",
+    "type": "streaming",
+    "file_type": "CSV",
+    "file_size": "500MB/min",
+    "gcs_file_path": "gs://input_load_test_streaming_job/*.csv",
+    "deid_template": "projects/dlp-dataflow-load-test/deidentifyTemplates/dlp-demo-deid-latest-1706594483019",
+    "raw_file_pattern": "gs://input_dlp_load_test_2/largecsv500MB.csv"
+  },
   {
     "name": "Batch1_csv",
     "type": "batch",
     "file_type": "CSV",
-    "file_size": "100MB",
-    "gcs_file_path": "gs://input_dlp_load_test_2/largecsv100MB.csv",
+    "file_size": "500MB",
+    "gcs_file_path": "gs://input_dlp_load_test_2/largecsv500MB.csv",
+    "deid_template": "projects/dlp-dataflow-load-test/deidentifyTemplates/dlp-demo-deid-latest-1706594483019"
+
+  },
+  {
+    "name": "Batch2_csv",
+    "type": "batch",
+    "file_type": "CSV",
+    "file_size": "1GB",
+    "gcs_file_path": "gs://input_dlp_load_test_2/largecsv1GB.csv",
+    "deid_template": "projects/dlp-dataflow-load-test/deidentifyTemplates/dlp-demo-deid-latest-1706594483019"
+
+  },
+  {
+    "name": "Batch3_csv",
+    "type": "batch",
+    "file_type": "CSV",
+    "file_size": "2GB",
+    "gcs_file_path": "gs://input_dlp_load_test_2/largecsv2GB.csv",
+    "deid_template": "projects/dlp-dataflow-load-test/deidentifyTemplates/dlp-demo-deid-latest-1706594483019"
+  },
+  {
+    "name": "Batch1_avro",
+    "type": "batch",
+    "file_type": "AVRO",
+    "file_size": "500MB",
+    "gcs_file_path": "gs://input_dlp_load_test_2/largeavro500MB.avro",
     "deid_template": "projects/dlp-dataflow-load-test/deidentifyTemplates/dlp-demo-deid-latest-1706594483019"
 
+  },
+  {
+    "name": "Batch2_avro",
+    "type": "batch",
+    "file_type": "AVRO",
+    "file_size": "750MB",
+    "gcs_file_path": "gs://input_dlp_load_test_2/largeavro750MB.avro",
+    "deid_template": "projects/dlp-dataflow-load-test/deidentifyTemplates/dlp-demo-deid-latest-1706594483019"
+  },
+  {
+    "name": "Batch3_avro",
+    "type": "batch",
+    "file_type": "AVRO",
+    "file_size": "1500MB",
+    "gcs_file_path": "gs://input_dlp_load_test_2/largeavro1500MB.avro",
+    "deid_template": "projects/dlp-dataflow-load-test/deidentifyTemplates/dlp-demo-deid-latest-1706594483019"
   }
 ]