diff --git a/Makefile b/Makefile index 2861c664..db75a22d 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,8 @@ SHELL=/bin/bash -o pipefail -ifndef DEPLOYMENT_ENVIRONMENT -$(error Please run "source environment" in the repo root directory before running make commands) -endif - -init-tf: - -rm -f $(TF_DATA_DIR)/*.tfstate - mkdir -p $(TF_DATA_DIR) - jq -n ".region=\"us-west-2\" | .bucket=env.TF_S3_BUCKET | .key=env.APP_NAME+env.DEPLOYMENT_ENVIRONMENT" > $(TF_DATA_DIR)/aws_config.json - terraform init - -deploy: init-tf - @if [[ $(DEPLOYMENT_ENVIRONMENT) == staging && $$(git symbolic-ref --short HEAD) != staging ]]; then echo Please deploy staging from the staging branch; exit 1; fi - @if [[ $(DEPLOYMENT_ENVIRONMENT) == prod && $$(git symbolic-ref --short HEAD) != prod ]]; then echo Please deploy prod from the prod branch; exit 1; fi - TF_VAR_APP_NAME=$(APP_NAME) TF_VAR_DEPLOYMENT_ENVIRONMENT=$(DEPLOYMENT_ENVIRONMENT) TF_VAR_BATCH_SSH_PUBLIC_KEY='$(BATCH_SSH_PUBLIC_KEY)' terraform apply - deploy-mock: aws ssm put-parameter --name /mock-aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id --value ami-12345678 --type String --endpoint-url http://localhost:9000 - cp test/mock.tf .; unset TF_CLI_ARGS_init; terraform init; terraform apply --auto-approve + cp test/mock.tf .; unset TF_CLI_ARGS_init; terraform init; TF_VAR_mock=true TF_VAR_app_name=swipe-test TF_VAR_batch_ec2_instance_types='["optimal"]' terraform apply --auto-approve $(TFSTATE_FILE): terraform state pull > $(TFSTATE_FILE) @@ -35,6 +20,6 @@ test: python -m unittest discover . get-logs: - aegea logs --start-time=-5m --no-export /aws/lambda/$(APP_NAME)-$(DEPLOYMENT_ENVIRONMENT) + aegea logs --start-time=-5m --no-export /aws/lambda/$(app_name) .PHONY: deploy init-tf lint format test diff --git a/environment b/environment deleted file mode 100644 index e8c780fc..00000000 --- a/environment +++ /dev/null @@ -1,30 +0,0 @@ -# Resolve the location of this file and set APP_HOME to the root -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done -export APP_HOME="$(cd -P "$(dirname "$SOURCE")" && pwd)" - -set -a -APP_NAME=swipe -DEPLOYMENT_ENVIRONMENT="${DEPLOYMENT_ENVIRONMENT:-dev}" -TF_DATA_DIR=".terraform.${DEPLOYMENT_ENVIRONMENT}" -AWS_DEFAULT_REGION=$(aws configure get region) -AWS_ACCOUNT_ID=$(aws sts get-caller-identity | jq -r .Account) -TFSTATE_FILE="${TF_DATA_DIR}/remote.tfstate" -EXPECT_ALIAS=${EXPECT_AWS_ACCOUNT_ALIAS:-czi-comp-bio} -AWS_ACCOUNT_ALIAS=$(aws iam list-account-aliases | jq -r '.AccountAliases[0]') -if [[ $AWS_ACCOUNT_ALIAS != $EXPECT_ALIAS ]]; then - echo "ERROR: Expected to be in AWS account $EXPECT_ALIAS, but got $AWS_ACCOUNT_ALIAS" 1>&2 - echo "ERROR: If this is expected, set the EXPECT_AWS_ACCOUNT_ALIAS environment variable" 1>&2 - unset APP_HOME APP_NAME DEPLOYMENT_ENVIRONMENT TF_DATA_DIR TFSTATE_FILE - return -fi - -TF_CLI_ARGS_output="--state ${TFSTATE_FILE}" -TF_CLI_ARGS_init="--backend-config ${APP_HOME}/${TF_DATA_DIR}/aws_config.json" - -# See https://github.com/terraform-providers/terraform-provider-aws/issues/1184 -AWS_SDK_LOAD_CONFIG=1 - -TF_S3_BUCKET=tfstate-$AWS_ACCOUNT_ID -BATCH_SSH_PUBLIC_KEY="$(cat ~/.ssh/${APP_NAME}-${DEPLOYMENT_ENVIRONMENT}.pub)" -set +a diff --git a/environment.test b/environment.test deleted file mode 100644 index 3197a1df..00000000 --- a/environment.test +++ /dev/null @@ -1,19 +0,0 @@ -set -a -APP_HOME="$(cd -P "$(dirname "$SOURCE")" && pwd)" -APP_NAME=swipe -DEPLOYMENT_ENVIRONMENT=test - -TF_DATA_DIR=".terraform.${DEPLOYMENT_ENVIRONMENT}" -TFSTATE_FILE="${TF_DATA_DIR}/remote.tfstate" - -TF_CLI_ARGS_output="--state ${TFSTATE_FILE}" -TF_CLI_ARGS_init="--backend-config ${APP_HOME}/${TF_DATA_DIR}/aws_config.json" - -# See https://github.com/terraform-providers/terraform-provider-aws/issues/1184 -AWS_SDK_LOAD_CONFIG=1 - -TF_VAR_APP_NAME=$APP_NAME -TF_VAR_DEPLOYMENT_ENVIRONMENT=$DEPLOYMENT_ENVIRONMENT -TF_VAR_BATCH_SSH_PUBLIC_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD3F6tyPEFEzV0LX3X8BsXdMsQz1x2cEikKDEY0aIj41qgxMCP/iteneqXSIFZBp5vizPvaoIR3Um9xK7PGoW8giupGn+EPuxIA4cDM4vzOqOkiMPhz5XK0whEjkVzTo4+S0puvDZuwIsdiW9mxhJc7tgBNL0cYlWSYVkz4G/fslNfRPW5mYAM49f4fhtxPb5ok4Q2Lg9dPKVHO/Bgeu5woMc7RY0p1ej6D4CKFE6lymSDJpW0YHX/wqE9+cfEauh7xZcG0q9t2ta6F6fmX0agvpFyZo8aFbXeUBr7osSCJNgvavWbM/06niWrOvYX2xwWdhXmXSrbX8ZbabVohBK41 email@example.com" -TF_VAR_batch_ec2_instance_types='["optimal"]' -set +a diff --git a/main.tf b/main.tf index f122c4a8..ae644fa0 100644 --- a/main.tf +++ b/main.tf @@ -8,22 +8,21 @@ terraform { } resource "aws_key_pair" "swipe_batch" { - key_name = "${var.APP_NAME}-${var.DEPLOYMENT_ENVIRONMENT}" - public_key = var.BATCH_SSH_PUBLIC_KEY - count = var.BATCH_SSH_PUBLIC_KEY != "" ? 1 : 0 + key_name = var.app_name + public_key = var.batch_ssh_public_key + count = var.batch_ssh_public_key != "" ? 1 : 0 } module "batch_subnet" { - source = "./terraform/modules/swipe-sfn-batch-subnet" - app_name = var.APP_NAME - deployment_environment = var.DEPLOYMENT_ENVIRONMENT - count = var.vpc_id == "" || length(var.batch_subnet_ids) == 0 ? 1 : 0 + source = "./terraform/modules/swipe-sfn-batch-subnet" + app_name = var.app_name + count = var.vpc_id == "" || length(var.batch_subnet_ids) == 0 ? 1 : 0 } module "batch_queue" { source = "./terraform/modules/swipe-sfn-batch-queue" - app_name = var.APP_NAME - deployment_environment = var.DEPLOYMENT_ENVIRONMENT + app_name = var.app_name + mock = var.mock batch_ssh_key_pair_id = length(aws_key_pair.swipe_batch) > 0 ? aws_key_pair.swipe_batch[0].id : "" batch_subnet_ids = length(module.batch_subnet) > 0 ? module.batch_subnet[0].batch_subnet_ids : var.batch_subnet_ids batch_ec2_instance_types = var.batch_ec2_instance_types @@ -39,12 +38,11 @@ locals { module "sfn" { source = "./terraform/modules/swipe-sfn" - app_name = var.APP_NAME - deployment_environment = var.DEPLOYMENT_ENVIRONMENT + app_name = var.app_name batch_job_docker_image = "ghcr.io/chanzuckerberg/swipe:${local.version}" batch_spot_job_queue_arn = module.batch_queue.batch_spot_job_queue_arn batch_ec2_job_queue_arn = module.batch_queue.batch_ec2_job_queue_arn - additional_s3_path = var.additional_s3_path + workspace_s3_prefix = var.workspace_s3_prefix job_policy_arns = var.job_policy_arns } diff --git a/scripts/build_docker_image.sh b/scripts/build_docker_image.sh deleted file mode 100755 index d0f82893..00000000 --- a/scripts/build_docker_image.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -if [[ $# != 3 ]]; then - echo "This script builds a Docker image for the given Dockerfile and uploads it to the Docker registry under the" - echo "given name and tag. The Docker registry is ECR by default, or GitHub Packages if running on GitHub Actions." - echo "Usage: $(basename $0) dockerfile image_name image_tag" - exit 1 -fi - -print_docker_build_log() { - if [[ -f docker_build.log ]]; then - echo "$0: Error while building docker image; begin Docker build log:" > /dev/stderr - cat docker_build.log > /dev/stderr - echo "$0: End Docker build log" > /dev/stderr - fi -} -trap print_docker_build_log ERR - -dockerfile=$1 -image_name=$2 -image_tag=$3 -echo "Checking if a Docker image exists for $dockerfile..." -if [[ -n ${GITHUB_ACTIONS:-} ]] && [[ ${DEPLOYMENT_ENVIRONMENT:-} == test ]]; then - echo $GITHUB_TOKEN | docker login docker.pkg.github.com --username $(dirname $GITHUB_REPOSITORY) --password-stdin - export DOCKER_IMAGE_URI="docker.pkg.github.com/${GITHUB_REPOSITORY}/${image_name}" - DOCKER_API="https://docker.pkg.github.com/v2/${GITHUB_REPOSITORY}/${image_name}" - if http -p Hh --check-status GET "${DOCKER_API}/manifests/${image_tag}" Authorization:"Bearer $GITHUB_TOKEN"; then - echo "Docker image found at ${DOCKER_IMAGE_URI}:${image_tag}, skipping build" - exit - fi -else - aws ecr get-login-password --region $AWS_DEFAULT_REGION \ - | docker login --username AWS --password-stdin \ - "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com" - export DOCKER_IMAGE_URI="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/${image_name}" - aws ecr create-repository --repository-name=$image_name || true - if aws ecr describe-images --repository-name=$image_name --image-ids=imageTag=$image_tag; then - echo "Docker image found at ${DOCKER_IMAGE_URI}:${image_tag}, skipping build" - exit - fi -fi -CACHE_FROM=""; docker pull "$DOCKER_IMAGE_URI" && CACHE_FROM="--cache-from $DOCKER_IMAGE_URI" -(docker build "$(dirname $dockerfile)" --tag "${DOCKER_IMAGE_URI}:${image_tag}" $CACHE_FROM || docker build "$(dirname $dockerfile)" --tag "${DOCKER_IMAGE_URI}:${image_tag}" --no-cache) > docker_build.log 2>&1 -docker push "${DOCKER_IMAGE_URI}:${image_tag}" diff --git a/scripts/build_docker_images.sh b/scripts/build_docker_images.sh deleted file mode 100755 index 9c8c6093..00000000 --- a/scripts/build_docker_images.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -if [[ -z ${APP_NAME:-} ]]; then - echo "Please source the environment file before running this script." - exit 1 -fi - -for dockerfile in ${APP_HOME}/docker/*/Dockerfile; do - DOCKER_IMAGE_NAME="idseq-$(basename $(dirname $dockerfile))" - DOCKERFILE_HASH="sha-$(cat $(dirname $dockerfile)/* | shasum | head -c 16)" - $(dirname $0)/build_docker_image.sh "$dockerfile" "$DOCKER_IMAGE_NAME" "$DOCKERFILE_HASH" -done diff --git a/scripts/init.sh b/scripts/init.sh index 0e6e6779..13ad2014 100755 --- a/scripts/init.sh +++ b/scripts/init.sh @@ -19,7 +19,7 @@ check_for_termination() { } put_metric() { - aws cloudwatch put-metric-data --metric-name $1 --namespace swipe-$DEPLOYMENT_ENVIRONMENT --unit Percent --value $2 --dimensions SFNCurrentState=$SFN_CURRENT_STATE + aws cloudwatch put-metric-data --metric-name $1 --namespace $APP_NAME --unit Percent --value $2 --dimensions SFNCurrentState=$SFN_CURRENT_STATE } put_metrics() { @@ -50,7 +50,7 @@ if [ -f /etc/profile ]; then source /etc/profile; fi miniwdl --version # Env vars that need to be forwarded to miniwdl's tasks in AWS Batch. -BATCH_SWIPE_ENVVARS="AWS_DEFAULT_REGION DEPLOYMENT_ENVIRONMENT AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" +BATCH_SWIPE_ENVVARS="AWS_DEFAULT_REGION AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" # set $WDL_PASSTHRU_ENVVARS to a list of space-separated env var names # to pass the values of those vars to miniwdl's task containers. PASSTHRU_VARS=( $BATCH_SWIPE_ENVVARS $WDL_PASSTHRU_ENVVARS ) diff --git a/scripts/init_ci_runner.sh b/scripts/init_ci_runner.sh index 973e6bec..76c9ec7a 100755 --- a/scripts/init_ci_runner.sh +++ b/scripts/init_ci_runner.sh @@ -25,6 +25,6 @@ pip install -r requirements-dev.txt set -x -source environment.test mkdir ~/.aws touch ~/.aws/credentials + diff --git a/scripts/run_sfn.py b/scripts/run_sfn.py index f3ba5c07..844de305 100755 --- a/scripts/run_sfn.py +++ b/scripts/run_sfn.py @@ -44,7 +44,6 @@ def format_log_level(level): timestamp = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") parser = argparse.ArgumentParser("run_sfn", description="Run an SFN-WDL workflow") -parser.add_argument("--environment", default=os.environ.get("DEPLOYMENT_ENVIRONMENT")) parser.add_argument("--sfn-name") parser.add_argument("--sfn-arn") parser.add_argument("--stages", nargs="+") @@ -58,7 +57,7 @@ def format_log_level(level): logs = boto3.client("logs") batch = boto3.client("batch") -app_slug = f"{os.environ['APP_NAME']}-{args.environment}" +app_name = os.environ["APP_NAME"] if args.sfn_name is None: args.sfn_name = "single-wdl" @@ -68,7 +67,7 @@ def format_log_level(level): if args.sfn_arn is None: args.sfn_arn = str(ARN(service="states", - resource=f"stateMachine:{app_slug}-{args.sfn_name}-1")) + resource=f"stateMachine:{app_name}-{args.sfn_name}-1")) args.sfn_input.setdefault("Input", { "Run": { @@ -81,7 +80,7 @@ def format_log_level(level): wdl_uri = args.wdl_uri args.sfn_input[f"{stage.upper()}_WDL_URI"] = wdl_uri -execution_name = f"{app_slug}-{timestamp}" +execution_name = f"{app_name}-{timestamp}" logger.info("Starting execution for %s", execution_name) res = sfn.start_execution(stateMachineArn=args.sfn_arn, diff --git a/terraform/iam_policy_templates/sfn_service.json b/terraform/iam_policy_templates/sfn_service.json index 96cc8622..b4aaee47 100644 --- a/terraform/iam_policy_templates/sfn_service.json +++ b/terraform/iam_policy_templates/sfn_service.json @@ -9,7 +9,7 @@ "states:StopExecution" ], "Resource": [ - "arn:aws:states:${AWS_DEFAULT_REGION}:${AWS_ACCOUNT_ID}:stateMachine:${APP_NAME}-${DEPLOYMENT_ENVIRONMENT}-*" + "arn:aws:states:${AWS_DEFAULT_REGION}:${AWS_ACCOUNT_ID}:stateMachine:${app_name}-*" ] }, { @@ -47,7 +47,7 @@ "lambda:InvokeFunction" ], "Resource": [ - "arn:aws:lambda:${AWS_DEFAULT_REGION}:${AWS_ACCOUNT_ID}:function:${APP_NAME}-${DEPLOYMENT_ENVIRONMENT}-*" + "arn:aws:lambda:${AWS_DEFAULT_REGION}:${AWS_ACCOUNT_ID}:function:${app_name}-*" ] }, { @@ -56,7 +56,7 @@ "sqs:SendMessage" ], "Resource": [ - "arn:aws:sqs:${AWS_DEFAULT_REGION}:${AWS_ACCOUNT_ID}:${APP_NAME}-${DEPLOYMENT_ENVIRONMENT}-*" + "arn:aws:sqs:${AWS_DEFAULT_REGION}:${AWS_ACCOUNT_ID}:${app_name}-*" ] } ] diff --git a/terraform/modules/sfn-io-helper-lambdas/app/app.py b/terraform/modules/sfn-io-helper-lambdas/app/app.py index 8d4f0932..98b174ca 100644 --- a/terraform/modules/sfn-io-helper-lambdas/app/app.py +++ b/terraform/modules/sfn-io-helper-lambdas/app/app.py @@ -80,7 +80,7 @@ def process_batch_event(event): def process_sfn_event(event): execution_arn = event.detail["executionArn"] - if f"{os.environ['APP_NAME']}-{os.environ['DEPLOYMENT_ENVIRONMENT']}" in execution_arn: + if os.environ["APP_NAME"] in execution_arn: batch_events.archive_sfn_history(execution_arn) reporting.emit_sfn_metric_values(event) diff --git a/terraform/modules/sfn-io-helper-lambdas/app/sfn_io_helper/reporting.py b/terraform/modules/sfn-io-helper-lambdas/app/sfn_io_helper/reporting.py index 739bfb02..f6de747e 100644 --- a/terraform/modules/sfn-io-helper-lambdas/app/sfn_io_helper/reporting.py +++ b/terraform/modules/sfn-io-helper-lambdas/app/sfn_io_helper/reporting.py @@ -14,20 +14,20 @@ def notify_failure(sfn_state): """Placeholder for sending a message to a queue for push based result processing""" -def emit_batch_metric_values(event, namespace=f"{os.environ['APP_NAME']}-{os.environ['DEPLOYMENT_ENVIRONMENT']}"): +def emit_batch_metric_values(event, namespace=os.environ["APP_NAME"]): """Emit CloudWatch metrics for a Batch event""" -def emit_sfn_metric_values(event, namespace=f"{os.environ['APP_NAME']}-{os.environ['DEPLOYMENT_ENVIRONMENT']}"): +def emit_sfn_metric_values(event, namespace=os.environ["APP_NAME"]): """Emit CloudWatch metrics for a SFN event""" -def emit_spot_interruption_metric(event, namespace=f"{os.environ['APP_NAME']}-{os.environ['DEPLOYMENT_ENVIRONMENT']}"): +def emit_spot_interruption_metric(event, namespace=os.environ["APP_NAME"]): """Emit a CloudWatch metric for an EC2 spot instance interruption event""" def emit_periodic_metrics( - namespace=f"{os.environ['APP_NAME']}-{os.environ['DEPLOYMENT_ENVIRONMENT']}", + namespace=os.environ["APP_NAME"], time_horizon=timedelta(days=1) ): """Emit CloudWatch metrics on a fixed schedule""" diff --git a/terraform/modules/sfn-io-helper-lambdas/app/sfn_io_helper/stage_io.py b/terraform/modules/sfn-io-helper-lambdas/app/sfn_io_helper/stage_io.py index 1dab5608..858942b6 100644 --- a/terraform/modules/sfn-io-helper-lambdas/app/sfn_io_helper/stage_io.py +++ b/terraform/modules/sfn-io-helper-lambdas/app/sfn_io_helper/stage_io.py @@ -84,7 +84,7 @@ def preprocess_sfn_input(sfn_state, aws_region, aws_account_id, state_machine_na ecr_repo = f"{aws_account_id}.dkr.ecr.{aws_region}.amazonaws.com" if "docker_image_id" not in stage_input: workflow_name, workflow_version = get_workflow_name(sfn_state).rsplit("-v", 1) - default_docker_image_id = f"{ecr_repo}/{os.environ['APP_NAME']}-{workflow_name}:v{workflow_version}" + default_docker_image_id = f"{ecr_repo}/{os.environ['app_name']}-{workflow_name}:v{workflow_version}" stage_input["docker_image_id"] = default_docker_image_id put_stage_input(sfn_state=sfn_state, stage=stage, stage_input=stage_input) return sfn_state diff --git a/terraform/modules/sfn-io-helper-lambdas/main.tf b/terraform/modules/sfn-io-helper-lambdas/main.tf index 93bbb975..658df8f6 100644 --- a/terraform/modules/sfn-io-helper-lambdas/main.tf +++ b/terraform/modules/sfn-io-helper-lambdas/main.tf @@ -21,7 +21,8 @@ locals { resource "aws_iam_role" "iam_role" { for_each = local.lambda_names - name = "${var.app_name}-${var.deployment_environment}-${each.key}" + + name = "${var.app_name}-${each.key}" assume_role_policy = jsonencode({ Version = "2012-10-17", @@ -47,7 +48,7 @@ resource "aws_iam_role_policy" "iam_role_policy" { policy = jsonencode({ Version : "2012-10-17", - Statement : [ + Statement : concat(var.workspace_s3_prefix == "" ? [] : [ { Effect : "Allow", Action : [ @@ -55,15 +56,11 @@ resource "aws_iam_role_policy" "iam_role_policy" { "s3:GetObject*", "s3:PutObject*" ], - Resource : compact([ - "arn:aws:s3:::${var.app_name}-${var.deployment_environment}-*", - "arn:aws:s3:::${var.app_name}-${var.deployment_environment}-*/*", - "arn:aws:s3:::sfn-wdl-dev", - "arn:aws:s3:::sfn-wdl-dev/*", - var.additional_s3_path != "" ? "arn:aws:s3:::${var.additional_s3_path}" : "", - var.additional_s3_path != "" ? "arn:aws:s3:::${var.additional_s3_path}/*" : "", - ]) - }, + Resource : [ + "arn:aws:s3:::${var.workspace_s3_prefix}", + "arn:aws:s3:::${var.workspace_s3_prefix}/*", + ] + }], [ { Effect : "Allow", Action : [ @@ -92,8 +89,8 @@ resource "aws_iam_role_policy" "iam_role_policy" { "states:GetExecutionHistory" ], Resource : [ - "arn:aws:states:${var.aws_region}:${var.aws_account_id}:stateMachine:${var.app_name}-${var.deployment_environment}-*", - "arn:aws:states:${var.aws_region}:${var.aws_account_id}:execution:${var.app_name}-${var.deployment_environment}-*" + "arn:aws:states:${var.aws_region}:${var.aws_account_id}:stateMachine:${var.app_name}-*", + "arn:aws:states:${var.aws_region}:${var.aws_account_id}:execution:${var.app_name}-*" ] }, { @@ -110,14 +107,14 @@ resource "aws_iam_role_policy" "iam_role_policy" { ], Resource : "arn:aws:logs:*:*:*" } - ] + ]) }) } resource "aws_lambda_function" "lambda" { for_each = local.lambda_names - function_name = "${var.app_name}-${var.deployment_environment}-${each.key}" + function_name = "${var.app_name}-${each.key}" runtime = "python3.8" handler = "app.${each.key}" memory_size = 256 @@ -130,17 +127,16 @@ resource "aws_lambda_function" "lambda" { environment { variables = { - APP_NAME = var.app_name - DEPLOYMENT_ENVIRONMENT = var.deployment_environment - RunSPOTMemoryDefault = "16000" - RunEC2MemoryDefault = "16000" - AWS_ENDPOINT_URL = var.deployment_environment == "test" ? "http://host.docker.internal:9000" : null + APP_NAME = var.app_name + RunSPOTMemoryDefault = "16000" + RunEC2MemoryDefault = "16000" + AWS_ENDPOINT_URL = var.mock ? "http://host.docker.internal:9000" : null } } } resource "aws_cloudwatch_event_rule" "process_batch_event" { - name = "${var.app_name}-${var.deployment_environment}-process_batch_event" + name = "${var.app_name}-process_batch_event" tags = var.tags event_pattern = jsonencode({ @@ -153,7 +149,7 @@ resource "aws_cloudwatch_event_rule" "process_batch_event" { } resource "aws_cloudwatch_event_rule" "process_sfn_event" { - name = "${var.app_name}-${var.deployment_environment}-process_sfn_event" + name = "${var.app_name}-process_sfn_event" tags = var.tags event_pattern = jsonencode({ "source" = ["aws.states"] }) } @@ -178,13 +174,13 @@ resource "aws_cloudwatch_event_rule" "report_spot_interruption" { resource "aws_cloudwatch_event_target" "process_batch_event" { rule = aws_cloudwatch_event_rule.process_batch_event.name - target_id = "${var.app_name}-${var.deployment_environment}-process_batch_event" + target_id = "${var.app_name}-process_batch_event" arn = aws_lambda_function.lambda["process_batch_event"].arn } resource "aws_cloudwatch_event_target" "process_sfn_event" { rule = aws_cloudwatch_event_rule.process_sfn_event.name - target_id = "${var.app_name}-${var.deployment_environment}-process_batch_event" + target_id = "${var.app_name}-process_batch_event" arn = aws_lambda_function.lambda["process_sfn_event"].arn } diff --git a/terraform/modules/sfn-io-helper-lambdas/variables.tf b/terraform/modules/sfn-io-helper-lambdas/variables.tf index 145d4b01..fd9bb103 100644 --- a/terraform/modules/sfn-io-helper-lambdas/variables.tf +++ b/terraform/modules/sfn-io-helper-lambdas/variables.tf @@ -2,8 +2,9 @@ variable "app_name" { type = string } -variable "deployment_environment" { - type = string +variable "mock" { + type = bool + default = false } variable "aws_region" { @@ -23,7 +24,7 @@ variable "tags" { default = {} } -variable "additional_s3_path" { +variable "workspace_s3_prefix" { type = string default = "" -} \ No newline at end of file +} diff --git a/terraform/modules/swipe-sfn-batch-job/main.tf b/terraform/modules/swipe-sfn-batch-job/main.tf index 77f05095..4f71d22e 100644 --- a/terraform/modules/swipe-sfn-batch-job/main.tf +++ b/terraform/modules/swipe-sfn-batch-job/main.tf @@ -3,8 +3,7 @@ data "aws_region" "current" {} data "aws_caller_identity" "current" {} locals { - app_slug = "${var.app_name}-${var.deployment_environment}" - ecr_url = "${data.aws_caller_identity.current.account_id}.dkr.ecr.${data.aws_region.current.name}.amazonaws.com" + ecr_url = "${data.aws_caller_identity.current.account_id}.dkr.ecr.${data.aws_region.current.name}.amazonaws.com" container_config = yamldecode(templatefile("${path.module}/batch_job_container_properties.yml", { app_name = var.app_name, batch_job_role_arn = aws_iam_role.swipe_batch_main_job.arn, @@ -16,7 +15,7 @@ locals { "WDL_OUTPUT_URI" = "Set this variable to the S3 URI where the WDL output JSON will be written", "SFN_EXECUTION_ID" = "Set this variable to the current step function execution ARN", "SFN_CURRENT_STATE" = "Set this variable to the current step function state name, like HostFilterEC2 or HostFilterSPOT", - "DEPLOYMENT_ENVIRONMENT" = var.deployment_environment, + "APP_NAME" = var.app_name "AWS_DEFAULT_REGION" = data.aws_region.current.name, "MINIWDL__S3PARCP__DOCKER_IMAGE" = var.batch_job_docker_image, "MINIWDL__DOWNLOAD_CACHE__PUT" = "true", @@ -31,7 +30,7 @@ locals { } resource "aws_iam_policy" "swipe_batch_main_job" { - name = "${local.app_slug}-batch-job" + name = "${var.app_name}-batch-job" policy = jsonencode({ Version : "2012-10-17", @@ -47,10 +46,10 @@ resource "aws_iam_policy" "swipe_batch_main_job" { Resource : compact([ "arn:aws:s3:::aegea-batch-jobs-${data.aws_caller_identity.current.account_id}", "arn:aws:s3:::aegea-batch-jobs-${data.aws_caller_identity.current.account_id}/*", - "arn:aws:s3:::sfn-wdl-dev", - "arn:aws:s3:::sfn-wdl-dev/*", - var.additional_s3_path != "" ? "arn:aws:s3:::${var.additional_s3_path}" : "", - var.additional_s3_path != "" ? "arn:aws:s3:::${var.additional_s3_path}/*" : "", + var.wdl_workflow_s3_prefix != "" ? "arn:aws:s3:::${var.wdl_workflow_s3_prefix}" : "", + var.wdl_workflow_s3_prefix != "" ? "arn:aws:s3:::${var.wdl_workflow_s3_prefix}/*" : "", + var.workspace_s3_prefix != "" ? "arn:aws:s3:::${var.workspace_s3_prefix}" : "", + var.workspace_s3_prefix != "" ? "arn:aws:s3:::${var.workspace_s3_prefix}/*" : "", ]) }, { @@ -60,8 +59,8 @@ resource "aws_iam_policy" "swipe_batch_main_job" { ], Resource : compact([ "arn:aws:s3:::aegea-batch-jobs-${data.aws_caller_identity.current.account_id}", - "arn:aws:s3:::sfn-wdl-dev", - var.additional_s3_path != "" ? format("arn:aws:s3:::%s", split("/", var.additional_s3_path)[0]) : "", + var.wdl_workflow_s3_prefix != "" ? format("arn:aws:s3:::%s", split("/", var.wdl_workflow_s3_prefix)[0]) : "", + var.workspace_s3_prefix != "" ? format("arn:aws:s3:::%s", split("/", var.workspace_s3_prefix)[0]) : "", ]) }, { @@ -76,7 +75,7 @@ resource "aws_iam_policy" "swipe_batch_main_job" { } resource "aws_iam_role" "swipe_batch_main_job" { - name = "${local.app_slug}-batch-job" + name = "${var.app_name}-batch-job" assume_role_policy = templatefile("${path.module}/../../iam_policy_templates/trust_policy.json", { trust_services = ["ecs-tasks"] }) @@ -100,7 +99,7 @@ resource "aws_iam_role_policy_attachment" "swipe_batch_main_job_ecr_readonly" { } resource "aws_batch_job_definition" "swipe_main" { - name = "${local.app_slug}-main" + name = "${var.app_name}-main" type = "container" tags = var.tags retry_strategy { diff --git a/terraform/modules/swipe-sfn-batch-job/variables.tf b/terraform/modules/swipe-sfn-batch-job/variables.tf index f054dcc7..4c4e1502 100644 --- a/terraform/modules/swipe-sfn-batch-job/variables.tf +++ b/terraform/modules/swipe-sfn-batch-job/variables.tf @@ -27,25 +27,25 @@ variable "batch_job_retry_attempts" { default = 1 } -variable "deployment_environment" { - description = "deployment environment: (test, dev, staging, prod, etc.)" - type = string - default = "dev" -} - variable "tags" { description = "Tags to apply to managed assets" type = map(string) default = {} } -variable "additional_s3_path" { +variable "workspace_s3_prefix" { type = string default = "" } +variable "wdl_workflow_s3_prefix" { + description = "S3 prefix where WDL workflows are stored, read permissions will be granted for this prefix" + type = string + default = "" +} + variable "job_policy_arns" { type = list(string) description = "Policy ARNs to attach to batch jobs" default = [] -} \ No newline at end of file +} diff --git a/terraform/modules/swipe-sfn-batch-queue/main.tf b/terraform/modules/swipe-sfn-batch-queue/main.tf index e8ebbff0..15c5e1e0 100644 --- a/terraform/modules/swipe-sfn-batch-queue/main.tf +++ b/terraform/modules/swipe-sfn-batch-queue/main.tf @@ -1,15 +1,14 @@ locals { - app_slug = "${var.app_name}-${var.deployment_environment}" launch_template_user_data_file = "${path.module}/container_instance_user_data" launch_template_user_data_hash = filemd5(local.launch_template_user_data_file) } data "aws_ssm_parameter" "swipe_batch_ami" { - name = "/${var.deployment_environment == "test" ? "mock-aws" : "aws"}/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id" + name = "/${var.mock ? "mock-aws" : "aws"}/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id" } resource "aws_iam_role" "swipe_batch_service_role" { - name = "${local.app_slug}-batch-service" + name = "${var.app_name}-batch-service" assume_role_policy = templatefile("${path.module}/../../iam_policy_templates/trust_policy.json", { trust_services = ["batch"] }) @@ -22,7 +21,7 @@ resource "aws_iam_role_policy_attachment" "swipe_batch_service_role" { } resource "aws_iam_role" "swipe_batch_spot_fleet_service_role" { - name = "${local.app_slug}-batch-spot-fleet-service" + name = "${var.app_name}-batch-spot-fleet-service" assume_role_policy = templatefile("${path.module}/../../iam_policy_templates/trust_policy.json", { trust_services = ["spotfleet"] }) @@ -35,7 +34,7 @@ resource "aws_iam_role_policy_attachment" "swipe_batch_spot_fleet_service_role" } resource "aws_iam_role" "swipe_batch_main_instance_role" { - name = "${local.app_slug}-batch-main-instance" + name = "${var.app_name}-batch-main-instance" assume_role_policy = templatefile("${path.module}/../../iam_policy_templates/trust_policy.json", { trust_services = ["ec2"] }) @@ -58,7 +57,7 @@ resource "aws_iam_role_policy_attachment" "swipe_batch_main_instance_role_ssm" { } resource "aws_iam_instance_profile" "swipe_batch_main" { - name = "${local.app_slug}-batch-main" + name = "${var.app_name}-batch-main" role = aws_iam_role.swipe_batch_main_instance_role.name } @@ -68,13 +67,13 @@ resource "aws_launch_template" "swipe_batch_main" { # The launch template resource increments its version when contents change, but the compute environment resource does # not recognize this change. We bind the launch template name to user data contents here, so any changes to user data # will cause the whole launch template to be replaced, forcing the compute environment to pick up the changes. - name = "${local.app_slug}-batch-main-${local.launch_template_user_data_hash}" + name = "${var.app_name}-batch-main-${local.launch_template_user_data_hash}" user_data = filebase64(local.launch_template_user_data_file) tags = var.tags } resource "aws_security_group" "swipe" { - name = local.app_slug + name = var.app_name vpc_id = var.vpc_id egress { from_port = 0 @@ -97,7 +96,7 @@ resource "aws_batch_compute_environment" "swipe_main" { } } - compute_environment_name_prefix = "${local.app_slug}-${each.key}-" + compute_environment_name_prefix = "${var.app_name}-${each.key}-" compute_resources { instance_role = aws_iam_instance_profile.swipe_batch_main.arn @@ -116,7 +115,7 @@ resource "aws_batch_compute_environment" "swipe_main" { bid_percentage = 100 spot_iam_fleet_role = aws_iam_role.swipe_batch_spot_fleet_service_role.arn tags = merge(var.tags, { - Name = "${var.app_name}-batch-${var.deployment_environment}-${each.key}" + Name = "${var.app_name}-batch-${each.key}" }) launch_template { @@ -140,7 +139,7 @@ resource "aws_batch_compute_environment" "swipe_main" { resource "aws_batch_job_queue" "swipe_main" { for_each = toset(["SPOT", "EC2"]) - name = "${local.app_slug}-main-${each.key}" + name = "${var.app_name}-main-${each.key}" state = "ENABLED" priority = 10 compute_environments = [ diff --git a/terraform/modules/swipe-sfn-batch-queue/variables.tf b/terraform/modules/swipe-sfn-batch-queue/variables.tf index da595440..a39b2a6a 100644 --- a/terraform/modules/swipe-sfn-batch-queue/variables.tf +++ b/terraform/modules/swipe-sfn-batch-queue/variables.tf @@ -4,6 +4,12 @@ variable "app_name" { default = "swipe" } +variable "mock" { + type = bool + default = false +} + + variable "vpc_id" { type = string default = "" @@ -12,7 +18,7 @@ variable "vpc_id" { variable "batch_ec2_instance_types" { description = "EC2 instance types to use for Batch EC2 compute environments" type = list(string) - default = ["r5d.4xlarge"] + default = ["r5d"] } variable "batch_subnet_ids" { @@ -27,12 +33,6 @@ variable "batch_ssh_key_pair_id" { default = "" } -variable "deployment_environment" { - description = "deployment environment: (test, dev, staging, prod, etc.)" - type = string - default = "dev" -} - variable "min_vcpus" { type = number description = "Minimum CPUs for this cluster" diff --git a/terraform/modules/swipe-sfn-batch-subnet/main.tf b/terraform/modules/swipe-sfn-batch-subnet/main.tf index 68a8715b..584c325a 100644 --- a/terraform/modules/swipe-sfn-batch-subnet/main.tf +++ b/terraform/modules/swipe-sfn-batch-subnet/main.tf @@ -1,7 +1,3 @@ -locals { - app_slug = "${var.app_name}-${var.deployment_environment}" -} - data "aws_availability_zones" "available" { state = "available" } @@ -10,14 +6,14 @@ resource "aws_vpc" "swipe" { cidr_block = var.cidr_block enable_dns_hostnames = true tags = merge(var.tags, { - Name = local.app_slug + Name = var.app_name }) } resource "aws_internet_gateway" "swipe" { vpc_id = aws_vpc.swipe.id tags = merge(var.tags, { - Name = local.app_slug + Name = var.app_name }) } @@ -34,6 +30,6 @@ resource "aws_subnet" "swipe" { cidr_block = cidrsubnet(aws_vpc.swipe.cidr_block, 8, index(data.aws_availability_zones.available.names, each.key)) map_public_ip_on_launch = true tags = merge(var.tags, { - Name = local.app_slug + Name = var.app_name }) } diff --git a/terraform/modules/swipe-sfn-batch-subnet/variables.tf b/terraform/modules/swipe-sfn-batch-subnet/variables.tf index 945b33cb..26425fff 100644 --- a/terraform/modules/swipe-sfn-batch-subnet/variables.tf +++ b/terraform/modules/swipe-sfn-batch-subnet/variables.tf @@ -10,14 +10,8 @@ variable "cidr_block" { default = "10.20.0.0/16" } -variable "deployment_environment" { - description = "deployment environment: (test, dev, staging, prod, etc.)" - type = string - default = "dev" -} - variable "tags" { description = "Tags to apply to managed assets" type = map(string) default = {} -} \ No newline at end of file +} diff --git a/terraform/modules/swipe-sfn/main.tf b/terraform/modules/swipe-sfn/main.tf index f776814d..29306a5b 100644 --- a/terraform/modules/swipe-sfn/main.tf +++ b/terraform/modules/swipe-sfn/main.tf @@ -1,5 +1,4 @@ locals { - app_slug = "${var.app_name}-${var.deployment_environment}" sfn_template_file = var.sfn_template_file == "" ? "${path.module}/sfn-templates/single-wdl.yml" : var.sfn_template_file } @@ -8,18 +7,17 @@ data "aws_region" "current" {} data "aws_caller_identity" "current" {} resource "aws_iam_policy" "swipe_sfn_service" { - name = "${local.app_slug}-sfn-service" + name = "${var.app_name}-sfn-service" policy = templatefile("${path.module}/../../iam_policy_templates/sfn_service.json", { - APP_NAME = var.app_name, - DEPLOYMENT_ENVIRONMENT = var.deployment_environment, - sfn_service_role_name = "${local.app_slug}-sfn-service", - AWS_DEFAULT_REGION = data.aws_region.current.name, - AWS_ACCOUNT_ID = data.aws_caller_identity.current.account_id, + app_name = var.app_name, + sfn_service_role_name = "${var.app_name}-sfn-service", + AWS_DEFAULT_REGION = data.aws_region.current.name, + AWS_ACCOUNT_ID = data.aws_caller_identity.current.account_id, }) } resource "aws_iam_role" "swipe_sfn_service" { - name = "${local.app_slug}-sfn-service" + name = "${var.app_name}-sfn-service" assume_role_policy = templatefile("${path.module}/../../iam_policy_templates/trust_policy.json", { trust_services = ["states"] }) @@ -36,25 +34,24 @@ module "batch_job" { app_name = var.app_name batch_job_docker_image = var.batch_job_docker_image batch_job_timeout_seconds = var.batch_job_timeout_seconds - deployment_environment = var.deployment_environment - additional_s3_path = var.additional_s3_path + workspace_s3_prefix = var.workspace_s3_prefix job_policy_arns = var.job_policy_arns tags = var.tags } module "sfn_io_helper" { - source = "../sfn-io-helper-lambdas" - app_name = var.app_name - aws_region = data.aws_region.current.name - aws_account_id = data.aws_caller_identity.current.account_id - deployment_environment = var.deployment_environment - batch_queue_arns = [var.batch_spot_job_queue_arn, var.batch_ec2_job_queue_arn] - additional_s3_path = var.additional_s3_path - tags = var.tags + source = "../sfn-io-helper-lambdas" + app_name = var.app_name + mock = var.mock + aws_region = data.aws_region.current.name + aws_account_id = data.aws_caller_identity.current.account_id + batch_queue_arns = [var.batch_spot_job_queue_arn, var.batch_ec2_job_queue_arn] + workspace_s3_prefix = var.workspace_s3_prefix + tags = var.tags } resource "aws_sfn_state_machine" "swipe_single_wdl" { - name = "${local.app_slug}-single-wdl" + name = "${var.app_name}-single-wdl" role_arn = aws_iam_role.swipe_sfn_service.arn definition = jsonencode(yamldecode(templatefile(local.sfn_template_file, { batch_spot_job_queue_arn = var.batch_spot_job_queue_arn, diff --git a/terraform/modules/swipe-sfn/variables.tf b/terraform/modules/swipe-sfn/variables.tf index a4eef93e..4b3ea502 100644 --- a/terraform/modules/swipe-sfn/variables.tf +++ b/terraform/modules/swipe-sfn/variables.tf @@ -4,6 +4,11 @@ variable "app_name" { default = "swipe" } +variable "mock" { + type = bool + default = false +} + variable "sfn_template_file" { description = "JSON file with SFN contents (default: terraform/modules/swipe-sfn/sfn-templates/single-wdl-1.yml, transcoded to JSON)" type = string @@ -31,19 +36,13 @@ variable "batch_ec2_job_queue_arn" { type = string } -variable "deployment_environment" { - description = "deployment environment: (test, dev, staging, prod, etc.)" - type = string - default = "dev" -} - variable "tags" { description = "Tags to apply to managed assets" type = map(string) default = {} } -variable "additional_s3_path" { +variable "workspace_s3_prefix" { description = "additional S3 path to be granted permission for" type = string default = "" diff --git a/variables.tf b/variables.tf index f2148f27..e70897cf 100644 --- a/variables.tf +++ b/variables.tf @@ -1,12 +1,13 @@ -variable "APP_NAME" { +variable "app_name" { type = string } -variable "DEPLOYMENT_ENVIRONMENT" { - type = string +variable "mock" { + type = bool + default = false } -variable "BATCH_SSH_PUBLIC_KEY" { +variable "batch_ssh_public_key" { type = string default = "" } @@ -58,8 +59,14 @@ variable "on_demand_desired_vcpus" { default = 0 } -variable "additional_s3_path" { - description = "additional S3 path to be granted permission for" +variable "workspace_s3_prefix" { + description = "S3 prefix where input, output, and log files will be stored, read and write permissions will be granted for this prefix" + type = string + default = "" +} + +variable "wdl_workflow_s3_prefix" { + description = "S3 prefix where WDL workflows are stored, read permissions will be granted for this prefix" type = string default = "" -} \ No newline at end of file +} diff --git a/version b/version index a4986ad8..f7857972 100644 --- a/version +++ b/version @@ -1 +1 @@ -v0.1.0-beta \ No newline at end of file +v0.2.0-beta \ No newline at end of file