Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add utilities for ecs. #73

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open

Add utilities for ecs. #73

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3513105
Add utilities for ecs.
rhoboat Sep 24, 2022
a8078c0
Allow logger injection.
rhoboat Sep 28, 2022
2452161
Update package alias.
rhoboat Sep 28, 2022
a7bc593
Fixed build regarding opts and logger.
rhoboat Sep 28, 2022
8f58963
Address build issues and more comments.
rhoboat Sep 28, 2022
a2d6f74
Update autoscaling code based on code review feedback.
rhoboat Sep 28, 2022
acbae38
Merge branch 'master' into feature/ecs
rhoboat Sep 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions awscommons/v2/auth.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/gruntwork-io/go-commons/errors"
"github.com/sirupsen/logrus"
)

const (
Expand All @@ -17,6 +18,8 @@ type Options struct {
Region string

Context context.Context

Logger *logrus.Entry
}

// NewOptions will create a new aws.Options struct that provides reasonable defaults for unspecified values.
Expand Down
75 changes: 47 additions & 28 deletions awscommons/v2/autoscaling.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,14 @@ import (

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/autoscaling"
autoscalingTypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types"
autoscaling_types "github.com/aws/aws-sdk-go-v2/service/autoscaling/types"
"github.com/gruntwork-io/go-commons/collections"
"github.com/gruntwork-io/go-commons/errors"
"github.com/gruntwork-io/go-commons/logging"
"github.com/gruntwork-io/go-commons/retry"
)

// GetAsgByName finds the Auto Scaling Group matching the given name. Returns an error if it cannot find a match.
func GetAsgByName(opts *Options, asgName string) (*autoscalingTypes.AutoScalingGroup, error) {
func GetAsgByName(opts *Options, asgName string) (*autoscaling_types.AutoScalingGroup, error) {
client, err := NewAutoScalingClient(opts)
if err != nil {
return nil, errors.WithStackTrace(err)
Expand All @@ -41,31 +40,35 @@ func ScaleUp(
maxRetries int,
sleepBetweenRetries time.Duration,
) ([]string, error) {
logger := logging.GetProjectLogger()

client, err := NewAutoScalingClient(opts)
if err != nil {
return nil, err
}

err = setAsgCapacity(client, opts, asgName, desiredCapacity)
if err != nil {
logger.Errorf("Failed to set ASG desired capacity to %d", desiredCapacity)
logger.Errorf("If the capacity is set in AWS, undo by lowering back to the original desired capacity. If the desired capacity is not yet set, triage the error message below and try again.")
if opts.Logger != nil {
opts.Logger.Errorf("Failed to set ASG desired capacity to %d", desiredCapacity)
opts.Logger.Errorf("If the capacity is set in AWS, undo by lowering back to the original desired capacity. If the desired capacity is not yet set, triage the error message below and try again.")
}
return nil, err
}

err = waitForCapacity(opts, asgName, maxRetries, sleepBetweenRetries)
if err != nil {
logger.Errorf("Timed out waiting for ASG to reach desired capacity.")
logger.Errorf("Undo by terminating all the new instances and trying again.")
if opts.Logger != nil {
opts.Logger.Errorf("Timed out waiting for ASG to reach desired capacity.")
opts.Logger.Errorf("Undo by terminating all the new instances and trying again.")
}
return nil, err
}

newInstanceIds, err := getLaunchedInstanceIds(opts, asgName, originalInstanceIds)
if err != nil {
logger.Errorf("Error retrieving information about the ASG.")
logger.Errorf("Undo by terminating all the new instances and trying again.")
if opts.Logger != nil {
opts.Logger.Errorf("Error retrieving information about the ASG.")
opts.Logger.Errorf("Undo by terminating all the new instances and trying again.")
}
return nil, err
}

Expand Down Expand Up @@ -93,8 +96,9 @@ func getLaunchedInstanceIds(opts *Options, asgName string, existingInstanceIds [
// setAsgCapacity sets the desired capacity on the auto scaling group. This will not wait for the ASG to expand or
// shrink to that size. See waitForCapacity.
func setAsgCapacity(client *autoscaling.Client, opts *Options, asgName string, desiredCapacity int32) error {
logger := logging.GetProjectLogger()
logger.Infof("Updating ASG %s desired capacity to %d.", asgName, desiredCapacity)
if opts.Logger != nil {
opts.Logger.Debugf("Updating ASG %s desired capacity to %d.", asgName, desiredCapacity)
}

input := &autoscaling.SetDesiredCapacityInput{
AutoScalingGroupName: aws.String(asgName),
Expand All @@ -105,15 +109,18 @@ func setAsgCapacity(client *autoscaling.Client, opts *Options, asgName string, d
return errors.WithStackTrace(err)
}

logger.Infof("Requested ASG %s desired capacity to be %d.", asgName, desiredCapacity)
if opts.Logger != nil {
opts.Logger.Debugf("Requested ASG %s desired capacity to be %d.", asgName, desiredCapacity)
}
return nil
}

// SetAsgMaxSize sets the max size on the auto scaling group. Note that updating the max size does not typically
// change the cluster size.
func SetAsgMaxSize(client *autoscaling.Client, opts *Options, asgName string, maxSize int32) error {
logger := logging.GetProjectLogger()
logger.Infof("Updating ASG %s max size to %d.", asgName, maxSize)
if opts.Logger != nil {
opts.Logger.Debugf("Updating ASG %s max size to %d.", asgName, maxSize)
}

input := &autoscaling.UpdateAutoScalingGroupInput{
AutoScalingGroupName: aws.String(asgName),
Expand All @@ -124,7 +131,9 @@ func SetAsgMaxSize(client *autoscaling.Client, opts *Options, asgName string, ma
return errors.WithStackTrace(err)
}

logger.Infof("Requested ASG %s max size to be %d.", asgName, maxSize)
if opts.Logger != nil {
opts.Logger.Debugf("Requested ASG %s max size to be %d.", asgName, maxSize)
}
return nil
}

Expand All @@ -135,15 +144,18 @@ func waitForCapacity(
maxRetries int,
sleepBetweenRetries time.Duration,
) error {
logger := logging.GetProjectLogger()
logger.Infof("Waiting for ASG %s to reach desired capacity.", asgName)
if opts.Logger != nil {
opts.Logger.Debugf("Waiting for ASG %s to reach desired capacity.", asgName)
}

err := retry.DoWithRetry(
logger.Logger,
opts.Logger.Logger,
"Waiting for desired capacity to be reached.",
maxRetries, sleepBetweenRetries,
func() error {
logger.Infof("Checking ASG %s capacity.", asgName)
if opts.Logger != nil {
opts.Logger.Debugf("Checking ASG %s capacity.", asgName)
}
asg, err := GetAsgByName(opts, asgName)
if err != nil {
// TODO: Should we retry this lookup or fail right away?
Expand All @@ -154,12 +166,16 @@ func waitForCapacity(
desiredCapacity := *asg.DesiredCapacity

if currentCapacity == desiredCapacity {
logger.Infof("ASG %s met desired capacity!", asgName)
if opts.Logger != nil {
opts.Logger.Debugf("ASG %s met desired capacity!", asgName)
}
return nil
}

logger.Infof("ASG %s not yet at desired capacity %d (current %d).", asgName, desiredCapacity, currentCapacity)
logger.Infof("Waiting for %s...", sleepBetweenRetries)
if opts.Logger != nil {
opts.Logger.Debugf("ASG %s not yet at desired capacity %d (current %d).", asgName, desiredCapacity, currentCapacity)
opts.Logger.Debugf("Waiting for %s...", sleepBetweenRetries)
}
return errors.WithStackTrace(fmt.Errorf("still waiting for desired capacity to be reached"))
},
)
Expand All @@ -177,8 +193,9 @@ func waitForCapacity(
// DetachInstances requests AWS to detach the instances, removing them from the ASG. It will also
// request to auto decrement the desired capacity.
func DetachInstances(opts *Options, asgName string, idList []string) error {
logger := logging.GetProjectLogger()
logger.Infof("Detaching %d instances from ASG %s", len(idList), asgName)
if opts.Logger != nil {
opts.Logger.Debugf("Detaching %d instances from ASG %s", len(idList), asgName)
}

client, err := NewAutoScalingClient(opts)
if err != nil {
Expand All @@ -198,12 +215,14 @@ func DetachInstances(opts *Options, asgName string, idList []string) error {
}
}

logger.Infof("Detached %d instances from ASG %s", len(idList), asgName)
if opts.Logger != nil {
opts.Logger.Debugf("Detached %d instances from ASG %s", len(idList), asgName)
}
return nil
}

// idsFromAsgInstances returns a list of the instance IDs given a list of instance representations from the ASG API.
func idsFromAsgInstances(instances []autoscalingTypes.Instance) []string {
func idsFromAsgInstances(instances []autoscaling_types.Instance) []string {
idList := []string{}
for _, inst := range instances {
idList = append(idList, aws.ToString(inst.InstanceId))
Expand Down
Loading