diff --git a/README.md b/README.md index 376577d..3d141ee 100644 --- a/README.md +++ b/README.md @@ -1 +1,23 @@ -# kroller \ No newline at end of file +# kroller + +``` +$ kroller + + _ _ _ +| | __ _ __ ___ | || | ___ _ __ +| |/ /| '__| / _ \ | || | / _ \| '__| +| < | | | (_) || || || __/| | +|_|\_\|_| \___/ |_||_| \___||_| + +USAGE + kroller + +SUBCOMMANDS + restart restart all rollout resources + drain drain node + +FLAGS + -kubeconfig ... kubeconfig file + -v false log verbose output +``` + diff --git a/go.mod b/go.mod index 3ced9c6..d65e6f7 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,12 @@ go 1.15 require ( github.com/CrowdSurge/banner v0.0.0-20140923200336-8c0e79dc5ff7 + github.com/aws/aws-sdk-go v1.38.21 github.com/fatih/color v1.10.0 github.com/googleapis/gnostic v0.5.4 // indirect github.com/imdario/mergo v0.3.11 // indirect github.com/peterbourgon/ff/v3 v3.0.0 + github.com/pkg/errors v0.9.1 github.com/rodaine/table v1.0.1 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect diff --git a/go.sum b/go.sum index 4f385c4..6d0a06a 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.38.21 h1:D08DXWI4QRaawLaW+OtsIEClOI90I6eheJs1GwXTQVI= +github.com/aws/aws-sdk-go v1.38.21/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -153,6 +155,9 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -193,6 +198,7 @@ github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterbourgon/ff/v3 v3.0.0 h1:eQzEmNahuOjQXfuegsKQTSTDbf4dNvr/eNLrmJhiH7M= github.com/peterbourgon/ff/v3 v3.0.0/go.mod h1:UILIFjRH5a/ar8TjXYLTkIvSvekZqPm5Eb/qbGk6CT0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -398,7 +404,6 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go index 4041a8c..62e9316 100644 --- a/main.go +++ b/main.go @@ -13,9 +13,11 @@ import ( func main() { rootCmd, cfg := cmd.NewRootCmd() restartCmd := cmd.NewRestartCmd(cfg) + drainCmd := cmd.NewDrainCmd(cfg) rootCmd.Subcommands = []*ffcli.Command{ restartCmd, + drainCmd, } if err := rootCmd.Parse(os.Args[1:]); err != nil { diff --git a/pkg/aws/asg.go b/pkg/aws/asg.go new file mode 100644 index 0000000..2eb8389 --- /dev/null +++ b/pkg/aws/asg.go @@ -0,0 +1,19 @@ +package aws + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/autoscaling" +) + +func (c *Client) TerminateInstance(instanceID string, decrDesiredCapacity bool) error { + auto := autoscaling.New(c.Session) + input := &autoscaling.TerminateInstanceInAutoScalingGroupInput{ + InstanceId: aws.String(instanceID), + ShouldDecrementDesiredCapacity: aws.Bool(decrDesiredCapacity), + } + _, err := auto.TerminateInstanceInAutoScalingGroup(input) + if err != nil { + return err + } + return nil +} diff --git a/pkg/aws/client.go b/pkg/aws/client.go new file mode 100644 index 0000000..0a9df04 --- /dev/null +++ b/pkg/aws/client.go @@ -0,0 +1,25 @@ +package aws + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" +) + +type Client struct { + Session *session.Session +} + +func NewClient(region string) (*Client, error) { + s, err := session.NewSession(&aws.Config{ + Region: aws.String(region), + }) + if err != nil { + return nil, err + } + + c := Client{ + Session: s, + } + + return &c, nil +} diff --git a/pkg/aws/ec2.go b/pkg/aws/ec2.go new file mode 100644 index 0000000..48bc857 --- /dev/null +++ b/pkg/aws/ec2.go @@ -0,0 +1,34 @@ +package aws + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" +) + +func (c *Client) GetInstanceID(nodeName string) (string, error) { + svc := ec2.New(c.Session) + + params := &ec2.DescribeInstancesInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("private-dns-name"), + Values: []*string{aws.String(nodeName)}, + }, + }, + } + + resp, err := svc.DescribeInstances(params) + if err != nil { + return "", err + } + + var instanceID string + for _, reservation := range resp.Reservations { + for _, instance := range reservation.Instances { + instanceID = *instance.InstanceId + break + } + break + } + return instanceID, nil +} diff --git a/pkg/cmd/drain.go b/pkg/cmd/drain.go new file mode 100644 index 0000000..0c3ed42 --- /dev/null +++ b/pkg/cmd/drain.go @@ -0,0 +1,210 @@ +package cmd + +import ( + "context" + "flag" + "fmt" + "time" + + "github.com/anarcher/kroller/pkg/aws" + "github.com/anarcher/kroller/pkg/kubernetes" + "github.com/anarcher/kroller/pkg/ui" + + "github.com/fatih/color" + "github.com/peterbourgon/ff/v3" + "github.com/peterbourgon/ff/v3/ffcli" + "github.com/rodaine/table" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type DrainConfig struct { + rootCfg *RootConfig + awsRegion string + gracePeriod time.Duration + node string + isTerminateNode bool + decrementDesiredCapacity bool +} + +func NewDrainCmd(rootCfg *RootConfig) *ffcli.Command { + cfg := &DrainConfig{ + rootCfg: rootCfg, + } + + fs := flag.NewFlagSet("kroller drain", flag.ExitOnError) + fs.String("config", "", "config file (optional)") + fs.StringVar(&cfg.awsRegion, "aws-region", "ap-northeast-2", "The region to use for node") + fs.DurationVar(&cfg.gracePeriod, "grace-period", (30 * time.Second), "Pod grace-period") + fs.StringVar(&cfg.node, "node", "", "The node that should drain") + fs.BoolVar(&cfg.isTerminateNode, "terminate-node", false, "Terminate the AWS instance in the autoscaling group") + fs.BoolVar(&cfg.decrementDesiredCapacity, "decr-desired-capacity", false, "Decrement desired capacity of the autoscaling group") + rootCfg.RegisterFlags(fs) + + c := &ffcli.Command{ + Name: "drain", + ShortUsage: "drain node", + ShortHelp: "drain node", + FlagSet: fs, + Options: []ff.Option{ + ff.WithEnvVarNoPrefix(), + ff.WithConfigFileFlag("config"), + ff.WithConfigFileParser(ff.PlainParser), + }, + Exec: cfg.Exec, + } + return c +} + +func (c *DrainConfig) Exec(ctx context.Context, args []string) error { + if c.node == "" { + return fmt.Errorf("node is required") + } + + if err := c.drainNode(ctx); err != nil { + return err + } + + if c.isTerminateNode == true { + if err := c.terminateNode(ctx); err != nil { + return err + } + } + + return nil +} + +func (c *DrainConfig) drainNode(ctx context.Context) error { + verbose := c.rootCfg.Verbose + kubeClient := c.rootCfg.KubeClient + + node, err := kubeClient.Node(ctx, c.node) + if err != nil { + return err + } + + allPods, err := kubeClient.PodsOnNode(ctx, c.node) + if err != nil { + return err + } + + pods := filterRollPods(allPods.Items) + + ui.PodList(pods) + fmt.Println("") + fmt.Printf(color.GreenString("Do you want to continue and drain?")) + ok, err := ui.AskForConfirm() + if err != nil { + return err + } + if !ok { + return nil + } + + if _, err := kubeClient.CordonNode(ctx, node); err != nil { + return err + } + + ui.Print("", verbose) + ui.PrintTitle("Cordon\n", verbose) + ui.Print(fmt.Sprintf("[✓] %s cordoned\n\n", node.ObjectMeta.Name), verbose) + + ui.PrintTitle("Evict Pods\n", verbose) + rollPods(ctx, kubeClient, pods, c.gracePeriod, verbose) + + return nil +} + +func (c *DrainConfig) terminateNode(ctx context.Context) error { + verbose := c.rootCfg.Verbose + + fmt.Println("") + fmt.Printf(color.RedString("Do you want to continue and terminate the node? ")) + ok, err := ui.AskForConfirm() + if err != nil { + return err + } + if !ok { + return nil + } + + ui.Print("", verbose) + ui.PrintTitle("Node termination:\n", verbose) + + client, err := aws.NewClient(c.awsRegion) + if err != nil { + return err + } + + instanceID, err := client.GetInstanceID(c.node) + if err != nil { + return err + } + + ui.Print(fmt.Sprintf("%-25s %s", "Private DNS:", c.node), verbose) + ui.Print(fmt.Sprintf("%-25s %s", "Instance ID:", instanceID), verbose) + ui.Print(fmt.Sprintf("Decrement desired capacity: %v", c.decrementDesiredCapacity), verbose) + + if err := client.TerminateInstance(instanceID, c.decrementDesiredCapacity); err != nil { + return err + } + + ui.Print("\n", verbose) + ui.Print("[✓] Node has been terminated!\n", true) + return nil +} + +func filterRollPods(pods []v1.Pod) []v1.Pod { + var res []v1.Pod + for _, p := range pods { + controllerRef := metav1.GetControllerOf(&p) + if controllerRef == nil { + continue + } + + if controllerRef.Kind == "DaemonSet" { + continue + } + res = append(res, p) + } + return res +} + +func rollPods(ctx context.Context, kubeClient *kubernetes.Client, pods []v1.Pod, gracePeriod time.Duration, verbose bool) error { + + graceP := int64(gracePeriod.Seconds()) + deleteOptions := metav1.DeleteOptions{GracePeriodSeconds: &graceP} + + fmt.Println("") + tbl := table.New(" ", "Evict pod", "New pod", "New node") + headerFmt := color.New(color.FgGreen, color.Underline).SprintfFunc() + columnFmt := color.New(color.FgYellow).SprintfFunc() + tbl.WithHeaderFormatter(headerFmt).WithFirstColumnFormatter(columnFmt) + + for _, pod := range pods { + err := kubeClient.DeletePod(ctx, pod, deleteOptions) + if err != nil { + return err + } + newPod, err := kubeClient.DetermineNewPod(ctx, pod) + if err != nil { + return err + } + if newPod != nil { + if err := kubeClient.WaitForPodToBeReady(ctx, newPod); err != nil { + return err + } + tbl.AddRow("[✓]", pod.Name, newPod.Name, newPod.Spec.NodeName) + } else { + tbl.AddRow("[✓]", pod.Name, "?", "?") + } + + if verbose { + fmt.Printf("Evicting pod: %s\n", pod.Name) + } + } + tbl.Print() + + return nil +} diff --git a/pkg/cmd/restart.go b/pkg/cmd/restart.go index 4127497..32b9e89 100644 --- a/pkg/cmd/restart.go +++ b/pkg/cmd/restart.go @@ -29,8 +29,10 @@ func NewRestartCmd(rootCfg *RootConfig) *ffcli.Command { rootCfg.RegisterFlags(fs) c := &ffcli.Command{ - Name: "restart", - FlagSet: fs, + Name: "restart", + ShortUsage: "restart all rollout resources (deployment,statefulset)", + ShortHelp: "restart all rollout resources", + FlagSet: fs, Options: []ff.Option{ ff.WithEnvVarNoPrefix(), ff.WithConfigFileFlag("config"), diff --git a/pkg/kubernetes/deploy.go b/pkg/kubernetes/deployment.go similarity index 100% rename from pkg/kubernetes/deploy.go rename to pkg/kubernetes/deployment.go diff --git a/pkg/kubernetes/node.go b/pkg/kubernetes/node.go new file mode 100644 index 0000000..8cb6171 --- /dev/null +++ b/pkg/kubernetes/node.go @@ -0,0 +1,42 @@ +package kubernetes + +import ( + "context" + + "github.com/pkg/errors" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" +) + +func (c *Client) Node(ctx context.Context, nodeName string) (*v1.Node, error) { + node, err := c.clientset.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrap(err, "failed to find the node") + } + return node, nil +} + +func (c *Client) PodsOnNode(ctx context.Context, nodeName string) (*v1.PodList, error) { + pods, err := c.clientset.CoreV1().Pods("").List(ctx, metav1.ListOptions{ + FieldSelector: fields.SelectorFromSet(fields.Set{ + "spec.nodeName": nodeName, + "status.phase": "Running", + }).String()}) + + if err != nil { + return nil, errors.Wrap(err, "failed to load pods on the node "+nodeName) + } + return pods, nil +} + +func (c *Client) CordonNode(ctx context.Context, node *v1.Node) (*v1.Node, error) { + node, err := c.clientset.CoreV1().Nodes().Get(ctx, node.ObjectMeta.Name, metav1.GetOptions{}) + node.Spec.Unschedulable = true + n, err := c.clientset.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{}) + if err != nil { + return nil, errors.Wrap(err, "failed to cordon node: "+node.Name) + } + return n, nil +} diff --git a/pkg/kubernetes/pod.go b/pkg/kubernetes/pod.go new file mode 100644 index 0000000..c81d21a --- /dev/null +++ b/pkg/kubernetes/pod.go @@ -0,0 +1,93 @@ +package kubernetes + +import ( + "context" + "sort" + "time" + + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func (c *Client) DeletePod(ctx context.Context, pod v1.Pod, deleteOptions metav1.DeleteOptions) error { + err := c.clientset.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, deleteOptions) + if err != nil { + return errors.Wrap(err, "failed to delete pod: "+pod.Name) + } + + return nil +} + +func (c *Client) DetermineNewPod(ctx context.Context, oldPod v1.Pod) (*v1.Pod, error) { + // Find all pods with the same labels: + pods, err := c.clientset.CoreV1().Pods(oldPod.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=" + oldPod.ObjectMeta.Labels["app"]}) + if len(pods.Items) == 0 { + return nil, nil + } + + if err != nil { + return nil, errors.Wrap(err, "failed to list pods with labels: "+"app="+oldPod.ObjectMeta.Labels["app"]) + } + + // Sort Pods to find the latest one + sort.Slice(pods.Items, func(i, j int) bool { + return pods.Items[i].CreationTimestamp.Time.After(pods.Items[j].CreationTimestamp.Time) + }) + + newPod := pods.Items[0] + + return &newPod, nil +} + +func (c *Client) WaitForPodToBeReady(ctx context.Context, pod *v1.Pod) error { + watcher, err := c.clientset.CoreV1().Pods(pod.Namespace).Watch(ctx, metav1.ListOptions{ + FieldSelector: "metadata.name=" + pod.Name, + }) + if err != nil { + return errors.Wrap(err, "cannot create Pod status listener") + } + + for { + e := <-watcher.ResultChan() + if e.Object == nil { + return errors.Wrap(err, "cannot read object") + } + p, ok := e.Object.(*v1.Pod) + if !ok { + continue + } + + if p.Name != pod.Name { + continue + } + + if p.Status.Phase == "Running" { + for i := 0; i <= 30; i++ { + p, err = c.clientset.CoreV1().Pods(p.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) + if err != nil { + return errors.Wrap(err, "error retriveing status for new pod") + } + if all(p.Status.ContainerStatuses, func(status v1.ContainerStatus) bool { return status.Ready }) { + break + } + time.Sleep(1 * time.Second) + } + break + } + } + + watcher.Stop() + + return nil +} + +func all(vs []v1.ContainerStatus, f func(v1.ContainerStatus) bool) bool { + for _, v := range vs { + if !f(v) { + return false + } + } + return true +} diff --git a/pkg/ui/pod.go b/pkg/ui/pod.go new file mode 100644 index 0000000..004c6f5 --- /dev/null +++ b/pkg/ui/pod.go @@ -0,0 +1,21 @@ +package ui + +import ( + "github.com/fatih/color" + "github.com/rodaine/table" + v1 "k8s.io/api/core/v1" +) + +func PodList(pods []v1.Pod) { + headerFmt := color.New(color.FgGreen, color.Underline).SprintfFunc() + columnFmt := color.New(color.FgYellow).SprintfFunc() + + tbl := table.New("Namespace", "Name") + tbl.WithHeaderFormatter(headerFmt).WithFirstColumnFormatter(columnFmt) + + for _, p := range pods { + tbl.AddRow(p.Namespace, p.Name) + } + + tbl.Print() +}