add option to output status JSON instead name hardcoding (#100)

main.tf

@@ -65,6 +65,7 @@ module "sfn" {
   extra_env_vars                = var.extra_env_vars
   sqs_queues                    = var.sqs_queues
   call_cache                    = var.call_cache
+  output_status_json_files      = var.output_status_json_files
   tags                          = var.tags
 }
 

miniwdl-plugins/sfn_wdl/sfnwdl_miniwdl_plugin.py

@@ -137,17 +137,10 @@ def update_status_json(logger, task, run_ids, s3_wd_uri, entries):
         return
 
     try:
-        # Figure out workflow and step names:
-        # e.g. run_ids = ["host_filter", "call-validate_input"]
-        workflow_name = run_ids[0]
-        if workflow_name in (
-            "czid_host_filter",
-            "czid_non_host_alignment",
-            "czid_postprocess",
-            "czid_experimental",
-            "czid_long_read_mngs",
-            "swipe_test",
-        ):
+        if os.getenv("OUTPUT_STATUS_JSON_FILES") == "true":
+            # Figure out workflow and step names:
+            # e.g. run_ids = ["host_filter", "call-validate_input"]
+            workflow_name = run_ids[0]
             workflow_name = "_".join(workflow_name.split("_")[1:])
             # parse --step-name from the task command template. For historical reasons, the status JSON
             # keys use this name and it's not the same as the WDL task name.

terraform/modules/swipe-sfn-batch-job/main.tf

@@ -34,6 +34,7 @@ locals {
     "MINIWDL__DOWNLOAD_CACHE__DISABLE_PATTERNS" = "[\"s3://swipe-samples-*/*\"]",
     "DOWNLOAD_CACHE_MAX_GB"                     = "500",
     "WDL_PASSTHRU_ENVVARS"                      = join(" ", [for k, v in var.extra_env_vars : k]),
+    "OUTPUT_STATUS_JSON_FILES"                  = tostring(var.output_status_json_files)
   })
   container_env_vars     = { "environment" : [for k in sort(keys(local.batch_env_vars)) : { "name" : k, "value" : local.batch_env_vars[k] }] }
   final_container_config = merge(local.container_config, local.container_env_vars)

terraform/modules/swipe-sfn-batch-job/variables.tf

@@ -36,6 +36,12 @@ variable "call_cache" {
   type        = bool
 }
 
+variable "output_status_json_files" {
+  description = "If set to true, swipe will upload a JSON file with the status and possible description of each task during execution, useful for progress reporting"
+  type        = bool
+  default     = false
+}
+
 variable "tags" {
   description = "Tags to apply to managed assets"
   type        = map(string)

terraform/modules/swipe-sfn/main.tf

@@ -43,6 +43,7 @@ module "batch_job" {
   extra_env_vars            = var.extra_env_vars
   docker_network            = var.docker_network
   call_cache                = var.call_cache
+  output_status_json_files  = var.output_status_json_files
   tags                      = var.tags
 }
 

terraform/modules/swipe-sfn/variables.tf

@@ -75,6 +75,13 @@ variable "docker_network" {
   type        = string
 }
 
+variable "output_status_json_files" {
+  description = "If set to true, swipe will upload a JSON file with the status and possible description of each task during execution, useful for progress reporting"
+  type        = bool
+  default     = false
+}
+
+
 // Module Specific
 
 variable "batch_job_docker_image" {

test/terraform/localstack/main.tf

@@ -33,4 +33,6 @@ module "swipetest" {
   }
 
   workspace_s3_prefixes = ["swipe-test"]
+
+  output_status_json_files = true
 }

test/terraform/moto/main.tf

@@ -33,4 +33,6 @@ module "swipetest" {
   }
 
   workspace_s3_prefixes = ["swipe-test"]
+
+  output_status_json_files = true
 }

variables.tf

@@ -187,4 +187,10 @@ variable "user_data_parts" {
     content      = string
   }))
   default = []
-}
+}
+
+variable "output_status_json_files" {
+  description = "If set to true, swipe will upload a JSON file with the status and possible description of each task during execution, useful for progress reporting"
+  type        = bool
+  default     = false
+}

version

@@ -1 +1 @@
-v1.2.1
+v1.3.0