speed up build by paritioning (#150)

speed up build by paritioningNote the most elegant solution, but the best quick-win I could find with travis.


### Test Plan
* run travis

.travis.yml

@@ -19,28 +19,30 @@ install:
   # - chmod +x ~/bin/terraform-docs
   # awscli
   - pip install awscli --upgrade --user
-  # gotest
-  - go get github.com/rakyll/gotest
-  - go install github.com/rakyll/gotest
+  - aws configure set aws_access_key_id     $CI1_AWS_ACCESS_KEY_ID     --profile cztack-ci-1
+  - aws configure set aws_secret_access_key $CI1_AWS_SECRET_ACCESS_KEY --profile cztack-ci-1
+  - aws --profile cztack-ci-1 sts get-caller-identity
+  - aws configure set aws_access_key_id     $CI2_AWS_ACCESS_KEY_ID     --profile cztack-ci-2
+  - aws configure set aws_secret_access_key $CI2_AWS_SECRET_ACCESS_KEY --profile cztack-ci-2
+  - aws --profile cztack-ci-2 sts get-caller-identity
+
+script:
+  - travis_wait 45 make test-ci
 
-jobs:
-  include:
-    - stage: check
-      script: make lint
-    # terraform-docs is incompatible with 0.12 for now
-    # - stage: check
-    #   script: make check-docs
-    - stage: check
-      before_script:
-        - aws configure set aws_access_key_id     $CI1_AWS_ACCESS_KEY_ID     --profile cztack-ci-1
-        - aws configure set aws_secret_access_key $CI1_AWS_SECRET_ACCESS_KEY --profile cztack-ci-1
-        - aws --profile cztack-ci-1 sts get-caller-identity
-        - aws configure set aws_access_key_id     $CI2_AWS_ACCESS_KEY_ID     --profile cztack-ci-2
-        - aws configure set aws_secret_access_key $CI2_AWS_SECRET_ACCESS_KEY --profile cztack-ci-2
-        - aws --profile cztack-ci-2 sts get-caller-identity
-      script: travis_wait 45 make test
 env:
+  jobs:
+    - TEST_BUCKET_INDEX=0
+    - TEST_BUCKET_INDEX=1
+    - TEST_BUCKET_INDEX=2
+    - TEST_BUCKET_INDEX=3
+    - TEST_BUCKET_INDEX=4
+    - TEST_BUCKET_INDEX=5
+    - TEST_BUCKET_INDEX=6
+    - TEST_BUCKET_INDEX=7
+    - TEST_BUCKET_INDEX=8
+    - TEST_BUCKET_INDEX=9
   global:
+    - TEST_BUCKETS=10
     - secure: H6toE/cJTjXbp9QEONjA2wvWxIPIzPiX3VRY0r5C6YTYEMeujR/w8XW/HMrUFgK7DrD6td5vgpDVeoC84SmQ/KIyU/jEOJsjktxcsMK4Y/5Pbi5p9iK24ps7LUFLDGtYAnYiSOWd3lTDr1vRp/N1Es8VaSqyyRJwi5NwBFGjA0YoxiriIYe7WRCz9HRTiUSQ3PKZPJBLznqUrClvwVq9kZyI7zwyNmDYOrbXzKRPJfPCxHlWE/RsglXu0tSmuL6qxXVIhqp7ijRwJcCSMqpXYyYXPIHAm3b4dyXLSCzI3j+HwZqoTtnZFROMYrhVrgPlcntOe/NWMDu3UurE3ePEG6ghP/E5VD6xgcyiFDYvQV5f4ERPhmfurmOewESJdrtNQhepGutTaUX/gEzdu8rTaxc1fWFmOkscQOWCiPThqowdBP2kQtcrv660Z1PmtNuHmGyBS8j0hkkuyfWjX6YMf+egXpJSDT5kaEw+l6mn9yqL752hUdehKiBEroWqiv9eszFkFqPtBvNZO9vi+bRLeraq9lZJRHjS/T6LosfrLnXAo1X3Uqx79ruNFqLccADlsSqVMsSnyyIyv2d4uXvYTgPrqra1vbu7TQgjuMvwHXiLGt+h/ZLm8HFvSCFNAwzz3Ca+LxTN0TNpEHGmiqj0FoTfyuENvbAMkL4wyU7RprA=
     - secure: hqVsbSEMU78f48VHpaNO/v3WkjF46LFIVlTEVZukS8NPPccYKLgiguerdfFYHAnqzDScZRn6r9/BhpNf3sMeaOsFpzZ3qiw0VOlDjQcpN/IafP+3w1j2xCCPUz+FLlDmrnn/iKM2YAkvKm56EV3XIM3NZmbO1SKfoCR0gfqWUysIF5AAr+9u9jZUT+jDabzLfmOyf8TPihbFvwvl0ptGr/ZRU1c0zExg2XFC7UHf6z/xYwkvSM6xX8pFX5QYbfmise6WBDLv0w/NH2WGI8OJr90SkSJMoUcnnmLuByUeQcg2ITcOuClhdUAQl9tCuM+q1OqTJc/hHXHMxqIfobUf8UmTiIzv4zxNJ5QCOvG0EpDEEqvhn80dC+ja/FYSulktWqdR0g/8hrSBa7b+qIg0IjS76fkSqP9gjJS4Nm/1iw6KlTcXLVx4tUlMTlQjkg5BxLf6OAUVSG88GNx6p4jaEW9XAzoM39YUWHOZsWWIkBBO961fVXlztsvU9N8I4sfUJ6z9S4FCrO6AJQCoyvXqd2dXExik1aXBGF2pa4OV4+eR+xeh7UJalb3coVafiJbXmqSjOu2s5awvSR2ieyt255233oA68g7gHKyXNVmwx032qiA5uktC5WpXpuf+dXC68p5Yy6I7f6KMp0urmwDq8XVbQ9FUlNPunY5RpLKIvlo=
     - secure: QfL0ry3PeS2C4r/+HGEmKIzx16kv13gu0bYnEhKVKqTVo0jGXkqOCPgqt8KD70dT45irW9arHOyv2K9cO4sNKfPxhMHHV9QFPYpTY3NEpaHOBUjz+22KvputspRMOOzBas0BCX2rtXAEcuBmgnRFK28zIR2OOy8HTOI5ybjYW0rEF0D8DwwUqIID0AxqO9Har8dX9PzDf6sdbEojxEGEnVtQQ9oejpCHdTPfbw8Ga7KryacS+NkslkKKXU4NpaUkyDF+CUkOipH9EdKzFinD3JIQIHoAE0R6qhsJioLffFmKv95h8pr1xj2iWQg6tsuXaTPRj/xanQG65ZBsd7PaFHf8A6e0O6nEx8TIlmzU2GZd4ld1GAwSLdgIUN9wRhSW0I/7VPpnN6hUuEeoT2cXfsiOffVH9eILWIhznEJ2u2wfGG0QFZvj8te2gS4T8VtsSGepsfBHX3mkId0UwemG4pqjfPVmJ7lQsimSSss8k26OjQdTyDtt2T5/qhmGJStdNTxPGzpbbZheCSxXBGPxq5SuvIXjnpkBf0o2/LHx8QRsEQQ3iA3vyTH+P0C5nO0Vhm58RKrs+TwPyrm6ExQJUKLu11cCN+/8SaWArLq0zNsrVvdjKeYdvUlhXEeSPKHzCPyKkAWORxWBIDGgKYxURaNyQFCm2I6/zZFOb8YAbd0=

Makefile

@@ -10,20 +10,27 @@ export ROUTE53_ZONE_ID := Z2W9YC1AAOXX6B
 export WILDCARD_CERT_ARN := arn:aws:acm:us-west-2:119435350371:certificate/b13e8e24-6436-488d-990d-2cedee3e36dd
 export CLOUDFRONT_WILDCARD_CERT_ARN := arn:aws:acm:us-east-1:119435350371:certificate/99f1fb68-4469-4280-b452-7193c8c787ca
 export ACCOUNT_ID := 119435350371
+export AWS_PROFILE=cztack-ci-1
+export AWS_SDK_LOAD_CONFIG=true
+export GOFLAGS=-mod=vendor
+export GO111MODULE=on
 
 all: clean fmt docs lint test
 
 setup: ## setup development dependencies
 	curl -L https://raw.githubusercontent.com/chanzuckerberg/bff/master/download.sh | sh
+.PHONY: setup
 
 release: ## run a release
 	bff bump
 	git push
+.PHONY: release
 
 fmt:
 	@for m in $(MODULES); do \
 		terraform fmt $m; \
 	done
+.PHONY: fmt
 
 lint:
 	@for m in $(MODULES); do \
@@ -33,23 +40,37 @@ lint:
 	@for m in $(MODULES); do \
 		ls $$m/*_test.go 2>/dev/null 1>/dev/null || (echo "no test(s) for $$m"; exit $$?); \
 	done
+.PHONY: lint
 
 docs:
 	@for m in $(MODULES); do \
 		pushd $$m; \
 		../scripts/update-readme.sh update; \
 		popd; \
 	done;
+.PHONY: docs
 
 check-docs:
 	@for m in $(MODULES); do \
 		pushd $$m; \
 		../scripts/update-readme.sh check || exit $$?; \
 		popd; \
 	done;
+.PHONY: check-docs
 
 clean:
 		rm **/*.tfstate*; true
+.PHONY: clean
 
 test: fmt
-	AWS_PROFILE=cztack-ci-1 AWS_SDK_LOAD_CONFIG=true gotest -count=1 -parallel 10 -test.timeout 45m $(TEST)
+	go test -count=1 -parallel 10 -test.timeout 45m $(TEST)
+.PHONY: test
+
+test-ci:
+	go list ./... | ./scripts/split-build $(TEST_BUCKETS) $(TEST_BUCKET_INDEX) | xargs go test -count=1 -parallel 10 -test.timeout 45m
+.PHONY: test
+
+deps:
+	go mod tidy
+	go mod vendor
+.PHONY: deps

aws-nuke.yml

@@ -0,0 +1,15 @@
+regions:
+  - us-west-2
+
+account-blacklist:
+  - "1"
+
+accounts:
+  "119435350371": {}
+
+resource-types:
+  # we can add to this over time
+  targets:
+    - RDSDBCluster
+    - RDSSnapshot
+    - RDSInstance

scripts/split-build

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+import os, sys
+
+buckets = int(sys.argv[1])
+bucket  = int(sys.argv[2])
+
+assert bucket < buckets
+
+b = []
+
+for i in range(buckets):
+    b.append([])
+
+for i, line in enumerate(sys.stdin):
+    for x in line.strip().split():
+        b[ i % buckets ].append(x)
+
+for pkg in b[bucket]:
+    print(pkg)