From 053ad65409d7eb57b5d6f61f372118349b5bab86 Mon Sep 17 00:00:00 2001 From: Cameron Showalter Date: Sat, 3 Aug 2024 10:15:12 -0800 Subject: [PATCH] Mass rewrite of config structure finally. Committing before I loose this much work --- ContainerManager/base_stack.py | 17 +- .../leaf_stack/NestedStacks/Container.py | 20 +- .../leaf_stack/NestedStacks/EcsAsg.py | 4 +- .../leaf_stack/NestedStacks/Efs.py | 16 +- .../leaf_stack/NestedStacks/SecurityGroups.py | 11 +- ContainerManager/leaf_stack/main.py | 5 +- ContainerManager/utils/config_loader.py | 83 +------- ContainerManager/utils/config_loader_v2.py | 191 ++++++++++++++++++ ContainerManager/utils/sns_subscriptions.py | 6 +- Examples/Minecraft-example.yaml | 3 +- Examples/Valheim-example.yaml | 3 +- README.md | 5 +- base-stack-config.yaml | 5 +- 13 files changed, 230 insertions(+), 139 deletions(-) create mode 100644 ContainerManager/utils/config_loader_v2.py diff --git a/ContainerManager/base_stack.py b/ContainerManager/base_stack.py index b22e119..7b6243c 100644 --- a/ContainerManager/base_stack.py +++ b/ContainerManager/base_stack.py @@ -43,7 +43,7 @@ def __init__( self, "Vpc", nat_gateways=0, - max_azs=config.get("MaxAZs", 1), + max_azs=config["Vpc"]["MaxAZs"], subnet_configuration=[ ec2.SubnetConfiguration( name=f"public-{construct_id}-sn", @@ -99,28 +99,23 @@ def __init__( }, ) ) - subscriptions = config.get("AlertSubscription", []) - add_sns_subscriptions(self, self.sns_notify_topic, subscriptions) + add_sns_subscriptions(self, self.sns_notify_topic, config["AlertSubscription"]) ##################### ### Route53 STUFF ### ##################### - if "Domain" not in config: - raise ValueError("Required key 'Domain' missing from config. See `./ContainerManager/README.md` on writing configs") - if "Name" not in config["Domain"]: - raise ValueError("Required key 'Domain.Name' missing from config. See `./ContainerManager/README.md` on writing configs") - - self.domain_name = str(config["Domain"]["Name"]).lower() + # domain_name is imported to other stacks, so save it to this one: + self.domain_name = config["Domain"]["Name"] self.root_hosted_zone_id = config["Domain"].get("HostedZoneId") - if self.root_hosted_zone_id: + if config["Domain"]["HostedZoneId"]: ## Import the existing Route53 Hosted Zone: # https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_route53.PublicHostedZoneAttributes.html self.root_hosted_zone = route53.PublicHostedZone.from_hosted_zone_attributes( self, "RootHostedZone", - hosted_zone_id=self.root_hosted_zone_id, + hosted_zone_id=config["Domain"]["HostedZoneId"], zone_name=self.domain_name, ) else: diff --git a/ContainerManager/leaf_stack/NestedStacks/Container.py b/ContainerManager/leaf_stack/NestedStacks/Container.py index 6439c90..e844765 100644 --- a/ContainerManager/leaf_stack/NestedStacks/Container.py +++ b/ContainerManager/leaf_stack/NestedStacks/Container.py @@ -40,22 +40,6 @@ def __init__( # task_role= permissions for *inside* the container ) - # Loop over each port and figure out what it wants: - port_mappings = [] - for port_info in container_config.get("Ports", []): - protocol, port = list(port_info.items())[0] - - - ### Create a list of mappings for the container: - port_mappings.append( - ecs.PortMapping( - host_port=port, - container_port=port, - # This will create something like: ecs.Protocol.TCP - protocol=getattr(ecs.Protocol, protocol.upper()), - ) - ) - ## Logs for the container: self.container_log_group = logs.LogGroup( self, @@ -74,13 +58,13 @@ def __init__( self.container = self.task_definition.add_container( container_id.title(), image=ecs.ContainerImage.from_registry(container_config["Image"]), - port_mappings=port_mappings, + port_mappings=container_config["Ports"], ## Hard limit. Won't ever go above this # memory_limit_mib=999999999, ## Soft limit. Container will go down to this if under heavy load, but can go higher memory_reservation_mib=4*1024, ## Add environment variables into the container here: - environment=container_config.get("Environment", {}), + environment=container_config["Environment"], ## Logging, straight from: # https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecs.LogDriver.html logging=ecs.LogDrivers.aws_logs( diff --git a/ContainerManager/leaf_stack/NestedStacks/EcsAsg.py b/ContainerManager/leaf_stack/NestedStacks/EcsAsg.py index 5f57ce6..ec022ad 100644 --- a/ContainerManager/leaf_stack/NestedStacks/EcsAsg.py +++ b/ContainerManager/leaf_stack/NestedStacks/EcsAsg.py @@ -36,7 +36,7 @@ def __init__( base_stack_sns_topic: sns.Topic, leaf_stack_sns_topic: sns.Topic, task_definition: ecs.Ec2TaskDefinition, - instance_type: str, + config: dict, sg_container_traffic: ec2.SecurityGroup, efs_file_system: efs.FileSystem, host_access_point: efs.AccessPoint, @@ -110,7 +110,7 @@ def __init__( self.launch_template = ec2.LaunchTemplate( self, "LaunchTemplate", - instance_type=ec2.InstanceType(instance_type), + instance_type=ec2.InstanceType(config["InstanceType"]), ## Needs to be an "EcsOptimized" image to register to the cluster # https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecs.EcsOptimizedImage.html machine_image=ecs.EcsOptimizedImage.amazon_linux2023(), diff --git a/ContainerManager/leaf_stack/NestedStacks/Efs.py b/ContainerManager/leaf_stack/NestedStacks/Efs.py index 1bc2b59..99576cb 100644 --- a/ContainerManager/leaf_stack/NestedStacks/Efs.py +++ b/ContainerManager/leaf_stack/NestedStacks/Efs.py @@ -31,19 +31,17 @@ def __init__( **kwargs, ) -> None: super().__init__(scope, "EfsNestedStack", **kwargs) - ## Persistent Storage: # https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_efs.FileSystem.html - efs_removal_policy = volume_config.get("RemovalPolicy", "RETAIN").upper() + if volume_config["KeepOnDelete"]: + efs_removal_policy = RemovalPolicy.RETAIN_ON_UPDATE_OR_DELETE + else: + efs_removal_policy = RemovalPolicy.DESTROY self.efs_file_system = efs.FileSystem( self, "Efs", vpc=vpc, - # Becomes something like `aws_cdk.RemovalPolicy.RETAIN`: - # TODO: EFS Doesn't support snapshot. Change config option to bool: - # https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.RemovalPolicy.html - # If `True`, set to `RETAIN_ON_UPDATE_OR_DELETE`, else `DESTROY`. - removal_policy=getattr(RemovalPolicy, efs_removal_policy), + removal_policy=efs_removal_policy, security_group=sg_efs_traffic, allow_anonymous_access=False, enable_automatic_backups=volume_config["EnableBackups"], @@ -73,9 +71,9 @@ def __init__( self.host_access_point = self.efs_file_system.add_access_point("efs-access-point-host", create_acl=ap_acl, path="/") ### Create mounts and attach them to the container: - for volume_info in volume_config.get("Paths", []): + for volume_info in volume_config["Paths"]: volume_path = volume_info["Path"] - read_only = volume_info.get("ReadOnly", False) + read_only = volume_info["ReadOnly"] ## Create a UNIQUE name, ID of game + (modified) path: # (Will be something like: `Minecraft-data` or `Valheim-opt-valheim`) volume_id = f"{container.container_name}{volume_path.replace('/','-')}" diff --git a/ContainerManager/leaf_stack/NestedStacks/SecurityGroups.py b/ContainerManager/leaf_stack/NestedStacks/SecurityGroups.py index d04e13a..03c2361 100644 --- a/ContainerManager/leaf_stack/NestedStacks/SecurityGroups.py +++ b/ContainerManager/leaf_stack/NestedStacks/SecurityGroups.py @@ -24,7 +24,7 @@ def __init__( leaf_construct_id: str, vpc: ec2.Vpc, container_id: str, - docker_ports_config: list, + config_container_ports: list, **kwargs, ) -> None: super().__init__(scope, "SecurityGroupsNestedStack", **kwargs) @@ -71,8 +71,13 @@ def __init__( ) # Loop over each port and figure out what it wants: - for port_info in docker_ports_config: - protocol, port = list(port_info.items())[0] + for port_mapping in config_container_ports: + ## Get the string "TCP" or "UDP": + # Starts from 'Protocol.TCP' + protocol = str(port_mapping.protocol).split(".")[1] + ## Get the port. Both 'host_port' and 'container_port' + # are the same. + port = port_mapping.host_port self.sg_container_traffic.connections.allow_from( ec2.Peer.any_ipv4(), diff --git a/ContainerManager/leaf_stack/main.py b/ContainerManager/leaf_stack/main.py index 26814b7..0ec25c1 100644 --- a/ContainerManager/leaf_stack/main.py +++ b/ContainerManager/leaf_stack/main.py @@ -73,8 +73,7 @@ def __init__( leaf_construct_id=construct_id, vpc=base_stack.vpc, container_id=container_id, - # sg_vpc_traffic=base_stack.sg_vpc_traffic, - docker_ports_config=config["Container"].get("Ports", []), + config_container_ports=config["Container"]["Ports"], ) ### All the info for the Container Stuff @@ -109,7 +108,7 @@ def __init__( base_stack_sns_topic=base_stack.sns_notify_topic, leaf_stack_sns_topic=self.sns_notify_topic, task_definition=self.container_nested_stack.task_definition, - instance_type=config["InstanceType"], + config=config["Ec2"], sg_container_traffic=self.sg_nested_stack.sg_container_traffic, efs_file_system=self.efs_nested_stack.efs_file_system, host_access_point=self.efs_nested_stack.host_access_point, diff --git a/ContainerManager/utils/config_loader.py b/ContainerManager/utils/config_loader.py index a9a86ba..4cb2b3c 100644 --- a/ContainerManager/utils/config_loader.py +++ b/ContainerManager/utils/config_loader.py @@ -4,20 +4,8 @@ # https://github.com/mkaranasou/pyaml_env from pyaml_env import parse_config -#################### -## HELPER METHODS ## -#################### -def check_missing(config: dict, required_vars: list) -> None: - missing_vars = [x for x in required_vars if x not in config] - if any(missing_vars): - raise RuntimeError(f"Missing environment vars: [{', '.join(missing_vars)}]") -####################### -## BASE CONFIG LOGIC ## -####################### -def load_base_config(path: str) -> dict: - # TODO: Parse - return parse_config(path) + ####################### ## LEAF CONFIG LOGIC ## @@ -26,84 +14,17 @@ def load_base_config(path: str) -> dict: # TODO: Re-write all of this, maybe create ec2.port objects instead of passing the dicts around all over the place. # (And maybe do with other dicts here too) -def parse_docker_ports(docker_ports_config: list) -> None: - for port_info in docker_ports_config: - ### Each port takes the format of {Protocol: Port}: - if len(port_info) != 1: - raise ValueError(f"Each port should have only one key-value pair. Got: {port_info}") - protocol, port = list(port_info.items())[0] - - # Check if the protocol is valid: - valid_protocols = ["TCP", "UDP"] - if protocol.upper() not in valid_protocols: - raise NotImplementedError(f"Protocol {protocol} is not supported. Only {valid_protocols} are supported for now.") - - # Check if the port is valid: - if not isinstance(port, int): - raise ValueError(f"Port {port} should be an integer.") -required_leaf_vars = [ - "InstanceType", - "Container", -] def load_leaf_config(path: str) -> dict: config = parse_config(path) - ## Check base-level keys in the config: - check_missing(config, required_leaf_vars) - - ## Check Container.* level keys in the config: - required_container_vars = [ - "Image", - "Ports", - ] - check_missing(config["Container"], required_container_vars) - parse_docker_ports(config["Container"]["Ports"]) - - ## Make the environment variables strings: - environment = config["Container"].get("Environment", {}) - environment = {key: str(val) for key, val in environment.items()} - config["Container"]["Environment"] = environment - - ###################### - ### VOLUMES CONFIG ### - ###################### - volume = config.get("Volume", {}) - if "RemovalPolicy" in volume: - volume["RemovalPolicy"] = volume["RemovalPolicy"].upper() - volume["EnableBackups"] = volume.get("EnableBackups", True) - - config["Volume"] = volume ####################### ### WATCHDOG CONFIG ### ####################### watchdog = config.get("Watchdog", {}) - if "MinutesWithoutConnections" not in watchdog: - watchdog["MinutesWithoutConnections"] = 5 - assert watchdog["MinutesWithoutConnections"] >= 2, "MinutesWithoutConnections must be at least 2." - - if "Type" not in watchdog: - using_tcp = any([list(x.keys())[0] == "TCP" for x in config["Container"]["Ports"]]) - using_udp = any([list(x.keys())[0] == "UDP" for x in config["Container"]["Ports"]]) - if using_tcp and using_udp: - raise ValueError("Watchdog type not specified, and could not be inferred from container ports. (Add Watchdog.Type)") - elif using_tcp: - watchdog["Type"] = "TCP" - elif using_udp: - watchdog["Type"] = "UDP" - else: - raise ValueError("Watchdog type not specified, and could not be inferred from container ports. (Add Watchdog.Type)") - - if watchdog["Type"] == "TCP": - if "TcpPort" not in watchdog: - if len(config["Container"]["Ports"]) != 1: - raise ValueError("Cannot infer TCP port from multiple ports. (Add Watchdog.TcpPort)") - watchdog["TcpPort"] = list(config["Container"]["Ports"][0].values())[0] - - elif watchdog["Type"] == "UDP": - pass + # Default changes depending on protocol used: if "Threshold" not in watchdog: diff --git a/ContainerManager/utils/config_loader_v2.py b/ContainerManager/utils/config_loader_v2.py new file mode 100644 index 0000000..2fbe29f --- /dev/null +++ b/ContainerManager/utils/config_loader_v2.py @@ -0,0 +1,191 @@ + +from aws_cdk import ( + aws_sns as sns, + aws_ecs as ecs, +) + +## Using this config for management, so you can have BOTH yaml and Env Vars: +# https://github.com/mkaranasou/pyaml_env +from pyaml_env import parse_config + +#################### +## HELPER METHODS ## +#################### +def raise_missing_key_error(key: str) -> None: + " Error telling user where to get help. " + raise ValueError(f"Required key '{key}' missing from config. See `./ContainerManager/README.md` on writing configs") + +def _parse_sns(config: dict) -> None: + if "AlertSubscription" not in config: + config["AlertSubscription"] = [] + new_config = [] + for subscription in config["AlertSubscription"]: + if len(subscription.items()) != 1: + raise ValueError(f"Each subscription should have only one key-value pair. Got: {subscription.items()}") + # The new key is the protocol cdk object itself: + protocol_str = list(subscription.keys())[0] + protocol = getattr(sns.SubscriptionProtocol, protocol_str.upper()) + new_config.append({ + protocol: subscription[protocol_str] + }) + config["AlertSubscription"] = new_config + +####################### +## BASE CONFIG LOGIC ## +####################### +def _parse_vpc(config: dict) -> None: + if "Vpc" not in config: + config["Vpc"] = {} + assert isinstance(config["Vpc"], dict) + config["Vpc"]["MaxAZs"] = config["Vpc"].get("MaxAZs", 1) + assert isinstance(config["Vpc"]["MaxAZs"], int) + +def _parse_domain(config: dict) -> None: + if "Domain" not in config: + config["Domain"] = {} + + # Check Domain.Name: + if "Name" not in config["Domain"]: + raise_missing_key_error("Domain.Name") + assert isinstance(config["Domain"]["Name"], str) + config["Domain"]["Name"] = config["Domain"]["Name"].lower() + + # Check Domain.HostedZoneId: + config["Domain"]["HostedZoneId"] = config["Domain"].get("HostedZoneId") + +def load_base_config(path: str) -> dict: + " For fact-checking/prepping the base config file. " + config = parse_config(path) + _parse_vpc(config) + _parse_domain(config) + _parse_sns(config) + return config + +####################### +## LEAF CONFIG LOGIC ## +####################### +def _parse_container(config: dict) -> None: + if "Container" not in config: + config["Container"] = {} + + ### Check Container.Image: + if "Image" not in config["Container"]: + raise_missing_key_error("Container.Image") + assert isinstance(config["Container"]["Image"], str) + + ### Parse Container.Ports: + if "Ports" not in config["Container"]: + raise_missing_key_error("Container.Ports") + assert isinstance(config["Container"]["Ports"], list) + new_ports = [] + valid_protocols = ["TCP", "UDP"] + # Loop over each port and figure out what it wants: + for port_info in config["Container"]["Ports"]: + protocol, port = list(port_info.items())[0] + assert protocol.upper() in valid_protocols, f"Protocol {protocol} is not supported. Only {valid_protocols} are supported for now." + assert isinstance(port, int) + + ### Create a list of mappings for the container: + new_ports.append( + # https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecs.PortMapping.html + ecs.PortMapping( + host_port=port, + container_port=port, + # This will create something like: ecs.Protocol.TCP + protocol=getattr(ecs.Protocol, protocol.upper()), + ) + ) + config["Container"]["Ports"] = new_ports + + ### Parse Container.Environment: + if "Environment" not in config["Container"]: + config["Container"]["Environment"] = {} + config["Container"]["Environment"] = { + key: str(val) for key, val in config["Container"]["Environment"].items() + } + + +def _parse_volume(config: dict) -> None: + if "Volume" not in config: + config["Volume"] = {} + + ### KeepOnDelete + if "KeepOnDelete" not in config["Volume"]: + config["Volume"]["KeepOnDelete"] = True + assert isinstance(config["Volume"]["KeepOnDelete"], bool) + + ### EnableBackups + if "EnableBackups" not in config["Volume"]: + config["Volume"]["EnableBackups"] = True + assert isinstance(config["Volume"]["EnableBackups"], bool) + + ### Paths + if "Paths" not in config["Volume"]: + config["Volume"]["Paths"] = [] + for path in config["Volume"]["Paths"]: + if "Path" not in path: + raise_missing_key_error("Volume.Paths[*].Path") + assert isinstance(path["Path"], str) + if "ReadOnly" not in path: + path["ReadOnly"] = False + assert isinstance(path["ReadOnly"], bool) + +def _parse_ec2(config: dict) -> None: + if "Ec2" not in config: + config["Ec2"] = {} + if "InstanceType" not in config["Ec2"]: + raise_missing_key_error("Ec2.InstanceType") + print(config["Ec2"]) + +def _parse_watchdog(config: dict) -> None: + if "Watchdog" not in config: + config["Watchdog"] = {} + + ### MinutesWithoutConnections + if "MinutesWithoutConnections" not in config["Watchdog"]: + config["Watchdog"]["MinutesWithoutConnections"] = 5 + assert isinstance(config["Watchdog"]["MinutesWithoutConnections"], int) + assert config["Watchdog"]["MinutesWithoutConnections"] >= 2, "Watchdog.MinutesWithoutConnections must be at least 2." + + ### Type + if "Type" not in config["Watchdog"]: + using_tcp = any([port.protocol == ecs.Protocol.TCP for port in config["Container"]["Ports"]]) + using_udp = any([port.protocol == ecs.Protocol.UDP for port in config["Container"]["Ports"]]) + # If you have both port types, no way to know which to use: + if using_tcp and using_udp: + raise_missing_key_error("Watchdog.Type") + # If just one, default to that: + elif using_tcp: + config["Watchdog"]["Type"] = "TCP" + elif using_udp: + config["Watchdog"]["Type"] = "UDP" + # If they don't have either, no idea what to do either: + else: + raise_missing_key_error("Watchdog.Type") + + ### Type - Extra Args + if config["Watchdog"]["Type"] == "TCP": + if "TcpPort" not in config["Watchdog"]: + tcp_ports = [port for port in config["Container"]["Ports"] if port.protocol == ecs.Protocol.TCP] + # If there's more than one TCP port: + if len(tcp_ports) != 1: + raise_missing_key_error("Watchdog.TcpPort") + # Both host_port and container_port are the same: + config["Watchdog"]["TcpPort"] = tcp_ports[0].host_port + elif config["Watchdog"]["Type"] == "UDP": + # No extra config options needed for UDP yet: + pass + + ### Threshold: + if "Threshold" not in config["Watchdog"]: + config["Watchdog"]["Threshold"] = {} + + +def load_leaf_config(path: str) -> dict: + config = parse_config(path) + _parse_container(config) + _parse_volume(config) + _parse_ec2(config) + _parse_watchdog(config) + _parse_sns(config) + return config diff --git a/ContainerManager/utils/sns_subscriptions.py b/ContainerManager/utils/sns_subscriptions.py index 14fde85..343c3e4 100644 --- a/ContainerManager/utils/sns_subscriptions.py +++ b/ContainerManager/utils/sns_subscriptions.py @@ -9,10 +9,8 @@ def add_sns_subscriptions(context, sns_topic: sns.Topic, subscriptions: dict) -> (Normally 'subscriptions' is the 'Alert Subscription' block from the config file) """ for subscription in subscriptions: - if len(subscription.items()) != 1: - raise ValueError(f"Each subscription should have only one key-value pair. Got: {subscription.items()}") - sub_type, address = list(subscription.items())[0] - protocol = getattr(sns.SubscriptionProtocol, sub_type.upper()) + # All of the error checking is in the config parser/loader: + protocol, address = list(subscription.items())[0] ## Email with a SNS Subscription: # https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_sns.Subscription.html sns.Subscription( diff --git a/Examples/Minecraft-example.yaml b/Examples/Minecraft-example.yaml index 53d10b4..3f1ea1b 100644 --- a/Examples/Minecraft-example.yaml +++ b/Examples/Minecraft-example.yaml @@ -1,5 +1,6 @@ -InstanceType: m5.large +Ec2: + InstanceType: m5.large Container: # Docs here: https://docker-minecraft-server.readthedocs.io/en/latest/ diff --git a/Examples/Valheim-example.yaml b/Examples/Valheim-example.yaml index 0170370..7627a36 100644 --- a/Examples/Valheim-example.yaml +++ b/Examples/Valheim-example.yaml @@ -1,5 +1,6 @@ -InstanceType: m5.large +Ec2: + InstanceType: m5.large Container: # Docs here: https://github.com/lloesche/valheim-server-docker diff --git a/README.md b/README.md index d12e901..a48a8c2 100644 --- a/README.md +++ b/README.md @@ -192,8 +192,7 @@ make aws-whoami ### Phase 2, Optimize and Cleanup -- Minor optimizations: - - Go through Cloudwatch log Groups, make sure everything has a retention policy by default, and removal policy DESTROY. +- DONE! ### Phase 3, Get ready for Production! @@ -208,8 +207,6 @@ make aws-whoami - Using pytest. Will also expand this to get timings of how long each part of the stack takes to spin up/down when someone connects. - [cdk-nag](https://github.com/cdklabs/cdk-nag) can flag some stuff -- Add Tags to stack/application, might help with analyzing costs and such. - - Add Generic stack tags, to help recognize the stack in the console. ### Continue after this diff --git a/base-stack-config.yaml b/base-stack-config.yaml index 83eab4e..2338c67 100644 --- a/base-stack-config.yaml +++ b/base-stack-config.yaml @@ -1,8 +1,9 @@ -MaxAZs: 1 +Vpc: + MaxAZs: 1 Domain: - Name: theamazingcameron.com + Name: TheAmazingCameron.com HostedZoneId: !ENV ${HOSTED_ZONE_ID} AlertSubscription: