end to end testing (#32)

.github/workflows/sfn-wdl-ci.yml

@@ -4,7 +4,6 @@ on:
   push
 
 env:
-  DEPLOYMENT_ENVIRONMENT: test
   LC_ALL: C.UTF-8
   LANG: C.UTF-8
   GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
@@ -38,12 +37,11 @@ jobs:
           key: ${{runner.os}}-cache
       - name: Run tests
         run: |
-          # This is required so tests can add entries
-          #   Tests need to add entries because we are mocking the s3 api and
-          #   we need to use s3's bucket DNS. When we make a test bucket
-          #   we must add an entry for it so we can use DNS locally.
-          sudo chmod 666 /etc/hosts
           source scripts/init_ci_runner.sh
+          source environment.test
+          # TODO: remove moto hack once change is merged
+          git clone https://github.com/morsecodist/moto.git ; cd moto ; git checkout tmorse-docker-networks ; docker build -t moto_local . ; cd .. ; rm -rf moto
+          docker build -t ghcr.io/chanzuckerberg/swipe:$(cat version) .
           scripts/run_mock_server.sh &
           for i in {1..60}; do if timeout 1 bash -c "echo > /dev/tcp/localhost/9000"; then break; elif [[ $i == 60 ]]; then exit 1; else sleep 1; fi; done
           make deploy-mock test

.gitignore

@@ -57,5 +57,7 @@ venv/
 
 /.terraform*
 
+sfn-io-helper-lambdas-tmp/
 
-sfn-io-helper-lambdas-tmp/
+mock.tf
+terraform.tfstate

Dockerfile

@@ -43,9 +43,9 @@ RUN apt-get -q install -y \
 RUN pip3 install miniwdl==${MINIWDL_VERSION} miniwdl-s3parcp==0.0.5 miniwdl-s3upload==0.0.8
 
 # TODO: generalize this plugin
-RUN pip3 install https://github.com/chanzuckerberg/miniwdl-plugins/archive/65e473bce33bc68c8c9a3049a2756e43e686c51e.zip#subdirectory=sfn-wdl
+RUN pip3 install https://github.com/chanzuckerberg/miniwdl-plugins/archive/v0.0.6.zip#subdirectory=sfn-wdl
 
-RUN curl -Ls https://github.com/chanzuckerberg/s3parcp/releases/download/v1.0.1-alpha/s3parcp_1.0.1-alpha_linux_amd64.tar.gz | tar -C /usr/bin -xz s3parcp
+RUN curl -Ls https://github.com/chanzuckerberg/s3parcp/releases/download/v1.0.3-alpha/s3parcp_1.0.3-alpha_linux_amd64.tar.gz | tar -C /usr/bin -xz s3parcp
 
 ADD https://raw.githubusercontent.com/chanzuckerberg/miniwdl/v${MINIWDL_VERSION}/examples/clean_download_cache.sh /usr/local/bin
 ADD scripts/init.sh /usr/local/bin

Makefile

@@ -1,12 +1,10 @@
 SHELL=/bin/bash -o pipefail
 
 deploy-mock:
+	rm terraform.tfstate || true
 	aws ssm put-parameter --name /mock-aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id --value ami-12345678 --type String --endpoint-url http://localhost:9000
 	cp test/mock.tf .; unset TF_CLI_ARGS_init; terraform init; TF_VAR_mock=true TF_VAR_app_name=swipe-test TF_VAR_batch_ec2_instance_types='["optimal"]' terraform apply --auto-approve
 
-$(TFSTATE_FILE):
-	terraform state pull > $(TFSTATE_FILE)
-
 lint:
 	flake8 .
 	yq . terraform/modules/swipe-sfn/default-wdl.yml > single-wdl.json

docker-compose.yml

@@ -3,17 +3,32 @@ version: "3.8"
 services:
   motoserver:
     container_name: motoserver
-    image: motoserver/moto
+    image: moto_local
+    environment:
+      - MOTO_DOCKER_NETWORK=awsnet
     ports:
       - "9000:5000"
     volumes:
       - "/var/run/docker.sock:/var/run/docker.sock"
+    networks:
+      awsnet:
+        aliases:
+          - awsnet
   stepfunctions_local:
     container_name: stepfunctions_local
     image: amazon/aws-stepfunctions-local
     environment:
-      - BATCH_ENDPOINT=http://motoserver:5000
-      - LAMBDA_ENDPOINT=http://motoserver:5000
+      - BATCH_ENDPOINT=http://awsnet:5000
+      - LAMBDA_ENDPOINT=http://awsnet:5000
       - AWS_ACCOUNT_ID=123456789012
     ports:
       - "8083:8083"
+    networks:
+      awsnet:
+        aliases:
+          - sfn.awsnet
+networks:
+  awsnet:
+    name: awsnet
+    driver: overlay
+    attachable: true

environment.test

@@ -0,0 +1,2 @@
+export AWS_REGION=us-east-1
+export AWS_DEFAULT_REGION=us-east-1

scripts/init.sh

@@ -4,6 +4,12 @@ if [ -f /etc/environment ]; then source /etc/environment; fi
 if [ -f /etc/default/locale ]; then source /etc/default/locale; else export LC_ALL=C.UTF-8 LANG=C.UTF-8; fi
 set +a
 
+if [ -n "${AWS_ENDPOINT_URL-}" ]; then
+  export aws="aws --endpoint-url ${AWS_ENDPOINT_URL}"
+else
+  export aws="aws"
+fi
+
 check_for_termination() {
   count=0
   while true; do
@@ -19,7 +25,7 @@ check_for_termination() {
 }
 
 put_metric() {
-  aws cloudwatch put-metric-data --metric-name $1 --namespace $APP_NAME --unit Percent --value $2 --dimensions SFNCurrentState=$SFN_CURRENT_STATE
+    $aws cloudwatch put-metric-data --metric-name $1 --namespace $APP_NAME --unit Percent --value $2 --dimensions SFNCurrentState=$SFN_CURRENT_STATE
 }
 
 put_metrics() {
@@ -58,8 +64,9 @@ PASSTHRU_ARGS=${PASSTHRU_VARS[@]/#/--env }
 
 set -euo pipefail
 export CURRENT_STATE=$(echo "$SFN_CURRENT_STATE" | sed -e s/SPOT// -e s/EC2//)
-aws s3 cp "$WDL_WORKFLOW_URI" .
-aws s3 cp "$WDL_INPUT_URI" wdl_input.json
+
+$aws s3 cp "$WDL_WORKFLOW_URI" .
+$aws s3 cp "$WDL_INPUT_URI" wdl_input.json
 
 handle_error() {
   OF=wdl_output.json;
@@ -70,7 +77,7 @@ handle_error() {
         tail -n 1 $(jq -r $EP $OF) > $OF;
       fi;
     fi;
-    aws s3 cp $OF "$WDL_OUTPUT_URI";
+    $aws s3 cp $OF "$WDL_OUTPUT_URI";
   fi
 }
 

scripts/init_ci_runner.sh

@@ -1,7 +1,6 @@
 # This script is sourced by GitHub Actions.
 
 set -a
-AWS_DEFAULT_OUTPUT=json
 DEBIAN_FRONTEND=noninteractive
 TERRAFORM_VERSION=0.15.0
 GH_CLI_VERSION=1.9.2
@@ -10,21 +9,12 @@ LANG=C.UTF-8
 TF_CLI_ARGS_apply="--auto-approve"
 set +a
 
-source /etc/profile
 sudo apt-get -qq update
-sudo apt-get -qq install -o=Dpkg::Use-Pty=0 --yes jq moreutils gettext build-essential python3-dev virtualenv zip unzip httpie git shellcheck ruby
+sudo apt-get -qq install -o=Dpkg::Use-Pty=0 --yes build-essential python3-dev unzip
 sudo gem install --quiet statelint
-curl -OLs https://github.com/cli/cli/releases/download/v${GH_CLI_VERSION}/gh_${GH_CLI_VERSION}_linux_amd64.deb
-sudo dpkg -i gh_${GH_CLI_VERSION}_linux_amd64.deb
 curl -OLs https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip
 sudo unzip -o terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/local/bin
-virtualenv --python=python3 .venv
-source .venv/bin/activate
-if [[ -d ~/.cache ]]; then sudo chown -R $(whoami) ~/.cache; fi
 pip install -r requirements-dev.txt
+docker swarm init
 
 set -x
-
-mkdir ~/.aws
-touch ~/.aws/credentials
-

terraform/modules/sfn-io-helper-lambdas/app/sfn_io_helper/stage_io.py

@@ -102,4 +102,6 @@ def preprocess_sfn_input(sfn_state, aws_region, aws_account_id, state_machine_na
             vcpu_key = stage + compute_env + "Vcpu"
             sfn_state.setdefault(vcpu_key, int(os.environ[vcpu_key + "Default"]))
 
+    link_outputs(sfn_state)
+
     return sfn_state

terraform/modules/sfn-io-helper-lambdas/main.tf

@@ -129,7 +129,7 @@ resource "aws_lambda_function" "lambda" {
   environment {
     variables = merge({
       APP_NAME         = var.app_name
-      AWS_ENDPOINT_URL = var.mock ? "http://host.docker.internal:9000" : null
+      AWS_ENDPOINT_URL = var.mock ? "http://awsnet:5000" : null
       }, {
       for stage, defaults in var.stage_memory_defaults : "${stage}SPOTMemoryDefault" => "${defaults.spot}"
       }, {

terraform/modules/swipe-sfn-batch-job/main.tf

@@ -25,7 +25,15 @@ locals {
     "DOWNLOAD_CACHE_MAX_GB"                     = "500",
     "WDL_PASSTHRU_ENVVARS"                      = join(" ", [for k, v in var.extra_env_vars : k]),
   })
-  container_env_vars     = { "environment" : [for k, v in local.batch_env_vars : { "name" : k, "value" : v }] }
+  mock_env_vars = var.mock ? {
+    "AWS_ACCESS_KEY_ID" : "role-account-id",
+    "AWS_SECRET_ACCESS_KEY" : "role-secret-key",
+    "AWS_SESSION_TOKEN" : "session-token",
+    "AWS_ENDPOINT_URL" : "http://awsnet:5000",
+    "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" : "container-credentials-relative-uri",
+    "S3PARCP_S3_URL" : "http://awsnet:5000",
+  } : {}
+  container_env_vars     = { "environment" : [for k, v in merge(local.batch_env_vars, local.mock_env_vars) : { "name" : k, "value" : v }] }
   final_container_config = merge(local.container_config, local.container_env_vars)
 }
 

terraform/modules/swipe-sfn-batch-job/variables.tf

@@ -5,6 +5,11 @@ variable "app_name" {
   description = "The name of your application, to be used as a namespace for all swipe managed assets"
 }
 
+variable "mock" {
+  type        = bool
+  description = "Set to true if applying to mock cloud environemnts for testing"
+}
+
 variable "job_policy_arns" {
   description = "Policy ARNs to attach to batch jobs"
   type        = list(string)

terraform/modules/swipe-sfn/main.tf

@@ -34,6 +34,7 @@ resource "aws_iam_role_policy_attachment" "swipe_sfn_service" {
 module "batch_job" {
   source                    = "../swipe-sfn-batch-job"
   app_name                  = var.app_name
+  mock                      = var.mock
   batch_job_docker_image    = var.batch_job_docker_image
   batch_job_timeout_seconds = var.batch_job_timeout_seconds
   workspace_s3_prefix       = var.workspace_s3_prefix

test/test_wdl.py

@@ -34,12 +34,12 @@
   }
 
   command <<<
-    cat {hello} > out.txt
-    cat world >> out.txt
+    cat ~{hello} > out.txt
+    echo world >> out.txt
   >>>
 
   output {
-    File out = out.txt
+    File out = "out.txt"
   }
 
   runtime {
@@ -48,8 +48,7 @@
 }
 """
 
-test_input = """
-hello
+test_input = """hello
 """
 
 
@@ -60,19 +59,19 @@ def setUp(self) -> None:
         self.test_bucket = self.s3.create_bucket(Bucket="swipe-test")
         self.lamb = boto3.client("lambda", endpoint_url="http://localhost:9000")
 
-    @unittest.skip("skipped due to networking within lambda and batch causing tests to fail")
     def test_simple_sfn_wdl_workflow(self):
         wdl_obj = self.test_bucket.Object("test-v1.0.0.wdl")
         wdl_obj.put(Body=test_wdl.encode())
         input_obj = self.test_bucket.Object("input.txt")
-        input_obj.put(Body=test_wdl.encode())
+        input_obj.put(Body=test_input.encode())
         output_prefix = "out"
         sfn_input: Dict[str, Any] = {
           "RUN_WDL_URI": f"s3://{wdl_obj.bucket_name}/{wdl_obj.key}",
           "OutputPrefix": f"s3://{input_obj.bucket_name}/{output_prefix}",
           "Input": {
               "Run": {
                   "hello": f"s3://{input_obj.bucket_name}/{input_obj.key}",
+                  "docker_image_id": "ubuntu",
               }
           }
         }
@@ -96,8 +95,9 @@ def test_simple_sfn_wdl_workflow(self):
             print(event, file=sys.stderr)
 
         assert description["status"] == "SUCCEEDED", description
-        outputs_obj = self.test_bucket.Object(f"{output_prefix}/output.txt")
-        assert outputs_obj.get()['Body'].read() == "hello\nworld"
+        outputs_obj = self.test_bucket.Object(f"{output_prefix}/test-1/out.txt")
+        output_text = outputs_obj.get()['Body'].read().decode()
+        assert output_text == "hello\nworld\n", output_text
 
 
 if __name__ == "__main__":

version

@@ -1 +1 @@
-v0.9.1-beta
+v0.9.2-beta