variables refactor (#21)

Makefile

@@ -1,23 +1,8 @@
 SHELL=/bin/bash -o pipefail
 
-ifndef DEPLOYMENT_ENVIRONMENT
-$(error Please run "source environment" in the repo root directory before running make commands)
-endif
-
-init-tf:
-	-rm -f $(TF_DATA_DIR)/*.tfstate
-	mkdir -p $(TF_DATA_DIR)
-	jq -n ".region=\"us-west-2\" | .bucket=env.TF_S3_BUCKET | .key=env.APP_NAME+env.DEPLOYMENT_ENVIRONMENT" > $(TF_DATA_DIR)/aws_config.json
-	terraform init
-
-deploy: init-tf
-	@if [[ $(DEPLOYMENT_ENVIRONMENT) == staging && $$(git symbolic-ref --short HEAD) != staging ]]; then echo Please deploy staging from the staging branch; exit 1; fi
-	@if [[ $(DEPLOYMENT_ENVIRONMENT) == prod && $$(git symbolic-ref --short HEAD) != prod ]]; then echo Please deploy prod from the prod branch; exit 1; fi
-	TF_VAR_APP_NAME=$(APP_NAME) TF_VAR_DEPLOYMENT_ENVIRONMENT=$(DEPLOYMENT_ENVIRONMENT) TF_VAR_BATCH_SSH_PUBLIC_KEY='$(BATCH_SSH_PUBLIC_KEY)' terraform apply
-
 deploy-mock:
 	aws ssm put-parameter --name /mock-aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id --value ami-12345678 --type String --endpoint-url http://localhost:9000
-	cp test/mock.tf .; unset TF_CLI_ARGS_init; terraform init; terraform apply --auto-approve
+	cp test/mock.tf .; unset TF_CLI_ARGS_init; terraform init; TF_VAR_mock=true TF_VAR_app_name=swipe-test TF_VAR_batch_ec2_instance_types='["optimal"]' terraform apply --auto-approve
 
 $(TFSTATE_FILE):
 	terraform state pull > $(TFSTATE_FILE)
@@ -35,6 +20,6 @@ test:
 	python -m unittest discover .
 
 get-logs:
-	aegea logs --start-time=-5m --no-export /aws/lambda/$(APP_NAME)-$(DEPLOYMENT_ENVIRONMENT)
+	aegea logs --start-time=-5m --no-export /aws/lambda/$(app_name)
 
 .PHONY: deploy init-tf lint format test

main.tf

@@ -8,22 +8,21 @@ terraform {
 }
 
 resource "aws_key_pair" "swipe_batch" {
-  key_name   = "${var.APP_NAME}-${var.DEPLOYMENT_ENVIRONMENT}"
-  public_key = var.BATCH_SSH_PUBLIC_KEY
-  count      = var.BATCH_SSH_PUBLIC_KEY != "" ? 1 : 0
+  key_name   = var.app_name
+  public_key = var.batch_ssh_public_key
+  count      = var.batch_ssh_public_key != "" ? 1 : 0
 }
 
 module "batch_subnet" {
-  source                 = "./terraform/modules/swipe-sfn-batch-subnet"
-  app_name               = var.APP_NAME
-  deployment_environment = var.DEPLOYMENT_ENVIRONMENT
-  count                  = var.vpc_id == "" || length(var.batch_subnet_ids) == 0 ? 1 : 0
+  source   = "./terraform/modules/swipe-sfn-batch-subnet"
+  app_name = var.app_name
+  count    = var.vpc_id == "" || length(var.batch_subnet_ids) == 0 ? 1 : 0
 }
 
 module "batch_queue" {
   source                   = "./terraform/modules/swipe-sfn-batch-queue"
-  app_name                 = var.APP_NAME
-  deployment_environment   = var.DEPLOYMENT_ENVIRONMENT
+  app_name                 = var.app_name
+  mock                     = var.mock
   batch_ssh_key_pair_id    = length(aws_key_pair.swipe_batch) > 0 ? aws_key_pair.swipe_batch[0].id : ""
   batch_subnet_ids         = length(module.batch_subnet) > 0 ? module.batch_subnet[0].batch_subnet_ids : var.batch_subnet_ids
   batch_ec2_instance_types = var.batch_ec2_instance_types
@@ -39,12 +38,11 @@ locals {
 
 module "sfn" {
   source                   = "./terraform/modules/swipe-sfn"
-  app_name                 = var.APP_NAME
-  deployment_environment   = var.DEPLOYMENT_ENVIRONMENT
+  app_name                 = var.app_name
   batch_job_docker_image   = "ghcr.io/chanzuckerberg/swipe:${local.version}"
   batch_spot_job_queue_arn = module.batch_queue.batch_spot_job_queue_arn
   batch_ec2_job_queue_arn  = module.batch_queue.batch_ec2_job_queue_arn
-  additional_s3_path       = var.additional_s3_path
+  workspace_s3_prefix      = var.workspace_s3_prefix
   job_policy_arns          = var.job_policy_arns
 }
 

scripts/init.sh

@@ -19,7 +19,7 @@ check_for_termination() {
 }
 
 put_metric() {
-  aws cloudwatch put-metric-data --metric-name $1 --namespace swipe-$DEPLOYMENT_ENVIRONMENT --unit Percent --value $2 --dimensions SFNCurrentState=$SFN_CURRENT_STATE
+  aws cloudwatch put-metric-data --metric-name $1 --namespace $APP_NAME --unit Percent --value $2 --dimensions SFNCurrentState=$SFN_CURRENT_STATE
 }
 
 put_metrics() {
@@ -50,7 +50,7 @@ if [ -f /etc/profile ]; then source /etc/profile; fi
 miniwdl --version
 
 # Env vars that need to be forwarded to miniwdl's tasks in AWS Batch.
-BATCH_SWIPE_ENVVARS="AWS_DEFAULT_REGION DEPLOYMENT_ENVIRONMENT AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"
+BATCH_SWIPE_ENVVARS="AWS_DEFAULT_REGION AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"
 # set $WDL_PASSTHRU_ENVVARS to a list of space-separated env var names
 # to pass the values of those vars to miniwdl's task containers.
 PASSTHRU_VARS=( $BATCH_SWIPE_ENVVARS $WDL_PASSTHRU_ENVVARS )

scripts/init_ci_runner.sh

@@ -25,6 +25,6 @@ pip install -r requirements-dev.txt
 
 set -x
 
-source environment.test
 mkdir ~/.aws
 touch ~/.aws/credentials
+

scripts/run_sfn.py

@@ -44,7 +44,6 @@ def format_log_level(level):
 timestamp = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
 
 parser = argparse.ArgumentParser("run_sfn", description="Run an SFN-WDL workflow")
-parser.add_argument("--environment", default=os.environ.get("DEPLOYMENT_ENVIRONMENT"))
 parser.add_argument("--sfn-name")
 parser.add_argument("--sfn-arn")
 parser.add_argument("--stages", nargs="+")
@@ -58,7 +57,7 @@ def format_log_level(level):
 logs = boto3.client("logs")
 batch = boto3.client("batch")
 
-app_slug = f"{os.environ['APP_NAME']}-{args.environment}"
+app_name = os.environ["APP_NAME"]
 
 if args.sfn_name is None:
     args.sfn_name = "single-wdl"
@@ -68,7 +67,7 @@ def format_log_level(level):
 
 if args.sfn_arn is None:
     args.sfn_arn = str(ARN(service="states",
-                           resource=f"stateMachine:{app_slug}-{args.sfn_name}-1"))
+                           resource=f"stateMachine:{app_name}-{args.sfn_name}-1"))
 
 args.sfn_input.setdefault("Input", {
     "Run": {
@@ -81,7 +80,7 @@ def format_log_level(level):
     wdl_uri = args.wdl_uri
     args.sfn_input[f"{stage.upper()}_WDL_URI"] = wdl_uri
 
-execution_name = f"{app_slug}-{timestamp}"
+execution_name = f"{app_name}-{timestamp}"
 
 logger.info("Starting execution for %s", execution_name)
 res = sfn.start_execution(stateMachineArn=args.sfn_arn,

terraform/iam_policy_templates/sfn_service.json

@@ -9,7 +9,7 @@
         "states:StopExecution"
       ],
       "Resource": [
-        "arn:aws:states:${AWS_DEFAULT_REGION}:${AWS_ACCOUNT_ID}:stateMachine:${APP_NAME}-${DEPLOYMENT_ENVIRONMENT}-*"
+        "arn:aws:states:${AWS_DEFAULT_REGION}:${AWS_ACCOUNT_ID}:stateMachine:${app_name}-*"
       ]
     },
     {
@@ -47,7 +47,7 @@
         "lambda:InvokeFunction"
       ],
       "Resource": [
-        "arn:aws:lambda:${AWS_DEFAULT_REGION}:${AWS_ACCOUNT_ID}:function:${APP_NAME}-${DEPLOYMENT_ENVIRONMENT}-*"
+        "arn:aws:lambda:${AWS_DEFAULT_REGION}:${AWS_ACCOUNT_ID}:function:${app_name}-*"
       ]
     },
     {
@@ -56,7 +56,7 @@
         "sqs:SendMessage"
       ],
       "Resource": [
-        "arn:aws:sqs:${AWS_DEFAULT_REGION}:${AWS_ACCOUNT_ID}:${APP_NAME}-${DEPLOYMENT_ENVIRONMENT}-*"
+        "arn:aws:sqs:${AWS_DEFAULT_REGION}:${AWS_ACCOUNT_ID}:${app_name}-*"
       ]
     }
   ]

terraform/modules/sfn-io-helper-lambdas/app/app.py

@@ -80,7 +80,7 @@ def process_batch_event(event):
 
 def process_sfn_event(event):
     execution_arn = event.detail["executionArn"]
-    if f"{os.environ['APP_NAME']}-{os.environ['DEPLOYMENT_ENVIRONMENT']}" in execution_arn:
+    if os.environ["APP_NAME"] in execution_arn:
         batch_events.archive_sfn_history(execution_arn)
 
     reporting.emit_sfn_metric_values(event)

terraform/modules/sfn-io-helper-lambdas/app/sfn_io_helper/reporting.py

@@ -14,20 +14,20 @@ def notify_failure(sfn_state):
     """Placeholder for sending a message to a queue for push based result processing"""
 
 
-def emit_batch_metric_values(event, namespace=f"{os.environ['APP_NAME']}-{os.environ['DEPLOYMENT_ENVIRONMENT']}"):
+def emit_batch_metric_values(event, namespace=os.environ["APP_NAME"]):
     """Emit CloudWatch metrics for a Batch event"""
 
 
-def emit_sfn_metric_values(event, namespace=f"{os.environ['APP_NAME']}-{os.environ['DEPLOYMENT_ENVIRONMENT']}"):
+def emit_sfn_metric_values(event, namespace=os.environ["APP_NAME"]):
     """Emit CloudWatch metrics for a SFN event"""
 
 
-def emit_spot_interruption_metric(event, namespace=f"{os.environ['APP_NAME']}-{os.environ['DEPLOYMENT_ENVIRONMENT']}"):
+def emit_spot_interruption_metric(event, namespace=os.environ["APP_NAME"]):
     """Emit a CloudWatch metric for an EC2 spot instance interruption event"""
 
 
 def emit_periodic_metrics(
-    namespace=f"{os.environ['APP_NAME']}-{os.environ['DEPLOYMENT_ENVIRONMENT']}",
+    namespace=os.environ["APP_NAME"],
     time_horizon=timedelta(days=1)
 ):
     """Emit CloudWatch metrics on a fixed schedule"""

terraform/modules/sfn-io-helper-lambdas/app/sfn_io_helper/stage_io.py

@@ -84,7 +84,7 @@ def preprocess_sfn_input(sfn_state, aws_region, aws_account_id, state_machine_na
         ecr_repo = f"{aws_account_id}.dkr.ecr.{aws_region}.amazonaws.com"
         if "docker_image_id" not in stage_input:
             workflow_name, workflow_version = get_workflow_name(sfn_state).rsplit("-v", 1)
-            default_docker_image_id = f"{ecr_repo}/{os.environ['APP_NAME']}-{workflow_name}:v{workflow_version}"
+            default_docker_image_id = f"{ecr_repo}/{os.environ['app_name']}-{workflow_name}:v{workflow_version}"
             stage_input["docker_image_id"] = default_docker_image_id
         put_stage_input(sfn_state=sfn_state, stage=stage, stage_input=stage_input)
     return sfn_state