Mass rewrite of config structure finally. Committing before I loose t…

…his much work

ContainerManager/base_stack.py

@@ -43,7 +43,7 @@ def __init__(
             self,
             "Vpc",
             nat_gateways=0,
-            max_azs=config.get("MaxAZs", 1),
+            max_azs=config["Vpc"]["MaxAZs"],
             subnet_configuration=[
                 ec2.SubnetConfiguration(
                     name=f"public-{construct_id}-sn",
@@ -99,28 +99,23 @@ def __init__(
                 },
             )
         )
-        subscriptions = config.get("AlertSubscription", [])
-        add_sns_subscriptions(self, self.sns_notify_topic, subscriptions)
+        add_sns_subscriptions(self, self.sns_notify_topic, config["AlertSubscription"])
 
 
         #####################
         ### Route53 STUFF ###
         #####################
-        if "Domain" not in config:
-            raise ValueError("Required key 'Domain' missing from config. See `./ContainerManager/README.md` on writing configs")
-        if "Name" not in config["Domain"]:
-            raise ValueError("Required key 'Domain.Name' missing from config. See `./ContainerManager/README.md` on writing configs")
-
-        self.domain_name = str(config["Domain"]["Name"]).lower()
+        # domain_name is imported to other stacks, so save it to this one:
+        self.domain_name = config["Domain"]["Name"]
         self.root_hosted_zone_id = config["Domain"].get("HostedZoneId")
 
-        if self.root_hosted_zone_id:
+        if config["Domain"]["HostedZoneId"]:
             ## Import the existing Route53 Hosted Zone:
             # https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_route53.PublicHostedZoneAttributes.html
             self.root_hosted_zone = route53.PublicHostedZone.from_hosted_zone_attributes(
                 self,
                 "RootHostedZone",
-                hosted_zone_id=self.root_hosted_zone_id,
+                hosted_zone_id=config["Domain"]["HostedZoneId"],
                 zone_name=self.domain_name,
             )
         else:

ContainerManager/leaf_stack/NestedStacks/Container.py

@@ -40,22 +40,6 @@ def __init__(
             # task_role= permissions for *inside* the container
         )
 
-        # Loop over each port and figure out what it wants:
-        port_mappings = []
-        for port_info in container_config.get("Ports", []):
-            protocol, port = list(port_info.items())[0]
-
-
-            ### Create a list of mappings for the container:
-            port_mappings.append(
-                ecs.PortMapping(
-                    host_port=port,
-                    container_port=port,
-                    # This will create something like: ecs.Protocol.TCP
-                    protocol=getattr(ecs.Protocol, protocol.upper()),
-                )
-            )
-
         ## Logs for the container:
         self.container_log_group = logs.LogGroup(
             self,
@@ -74,13 +58,13 @@ def __init__(
         self.container = self.task_definition.add_container(
             container_id.title(),
             image=ecs.ContainerImage.from_registry(container_config["Image"]),
-            port_mappings=port_mappings,
+            port_mappings=container_config["Ports"],
             ## Hard limit. Won't ever go above this
             # memory_limit_mib=999999999,
             ## Soft limit. Container will go down to this if under heavy load, but can go higher
             memory_reservation_mib=4*1024,
             ## Add environment variables into the container here:
-            environment=container_config.get("Environment", {}),
+            environment=container_config["Environment"],
             ## Logging, straight from:
             # https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecs.LogDriver.html
             logging=ecs.LogDrivers.aws_logs(

ContainerManager/leaf_stack/NestedStacks/EcsAsg.py

@@ -36,7 +36,7 @@ def __init__(
         base_stack_sns_topic: sns.Topic,
         leaf_stack_sns_topic: sns.Topic,
         task_definition: ecs.Ec2TaskDefinition,
-        instance_type: str,
+        config: dict,
         sg_container_traffic: ec2.SecurityGroup,
         efs_file_system: efs.FileSystem,
         host_access_point: efs.AccessPoint,
@@ -110,7 +110,7 @@ def __init__(
         self.launch_template = ec2.LaunchTemplate(
             self,
             "LaunchTemplate",
-            instance_type=ec2.InstanceType(instance_type),
+            instance_type=ec2.InstanceType(config["InstanceType"]),
             ## Needs to be an "EcsOptimized" image to register to the cluster
             # https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecs.EcsOptimizedImage.html
             machine_image=ecs.EcsOptimizedImage.amazon_linux2023(),

ContainerManager/leaf_stack/NestedStacks/Efs.py

@@ -31,19 +31,17 @@ def __init__(
         **kwargs,
     ) -> None:
         super().__init__(scope, "EfsNestedStack", **kwargs)
-
         ## Persistent Storage:
         # https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_efs.FileSystem.html
-        efs_removal_policy = volume_config.get("RemovalPolicy", "RETAIN").upper()
+        if volume_config["KeepOnDelete"]:
+            efs_removal_policy = RemovalPolicy.RETAIN_ON_UPDATE_OR_DELETE
+        else:
+            efs_removal_policy = RemovalPolicy.DESTROY
         self.efs_file_system = efs.FileSystem(
             self,
             "Efs",
             vpc=vpc,
-            # Becomes something like `aws_cdk.RemovalPolicy.RETAIN`:
-            # TODO: EFS Doesn't support snapshot. Change config option to bool:
-            #       https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.RemovalPolicy.html
-            #       If `True`, set to `RETAIN_ON_UPDATE_OR_DELETE`, else `DESTROY`.
-            removal_policy=getattr(RemovalPolicy, efs_removal_policy),
+            removal_policy=efs_removal_policy,
             security_group=sg_efs_traffic,
             allow_anonymous_access=False,
             enable_automatic_backups=volume_config["EnableBackups"],
@@ -73,9 +71,9 @@ def __init__(
         self.host_access_point = self.efs_file_system.add_access_point("efs-access-point-host", create_acl=ap_acl, path="/")
 
         ### Create mounts and attach them to the container:
-        for volume_info in volume_config.get("Paths", []):
+        for volume_info in volume_config["Paths"]:
             volume_path = volume_info["Path"]
-            read_only = volume_info.get("ReadOnly", False)
+            read_only = volume_info["ReadOnly"]
             ## Create a UNIQUE name, ID of game + (modified) path:
             #   (Will be something like: `Minecraft-data` or `Valheim-opt-valheim`)
             volume_id = f"{container.container_name}{volume_path.replace('/','-')}"

ContainerManager/leaf_stack/NestedStacks/SecurityGroups.py

@@ -24,7 +24,7 @@ def __init__(
         leaf_construct_id: str,
         vpc: ec2.Vpc,
         container_id: str,
-        docker_ports_config: list,
+        config_container_ports: list,
         **kwargs,
     ) -> None:
         super().__init__(scope, "SecurityGroupsNestedStack", **kwargs)
@@ -71,8 +71,13 @@ def __init__(
         )
 
         # Loop over each port and figure out what it wants:
-        for port_info in docker_ports_config:
-            protocol, port = list(port_info.items())[0]
+        for port_mapping in config_container_ports:
+            ## Get the string "TCP" or "UDP":
+            # Starts from 'Protocol.TCP'
+            protocol = str(port_mapping.protocol).split(".")[1]
+            ## Get the port. Both 'host_port' and 'container_port'
+            #   are the same.
+            port = port_mapping.host_port
 
             self.sg_container_traffic.connections.allow_from(
                 ec2.Peer.any_ipv4(),

ContainerManager/leaf_stack/main.py

@@ -73,8 +73,7 @@ def __init__(
             leaf_construct_id=construct_id,
             vpc=base_stack.vpc,
             container_id=container_id,
-            # sg_vpc_traffic=base_stack.sg_vpc_traffic,
-            docker_ports_config=config["Container"].get("Ports", []),
+            config_container_ports=config["Container"]["Ports"],
         )
 
         ### All the info for the Container Stuff
@@ -109,7 +108,7 @@ def __init__(
             base_stack_sns_topic=base_stack.sns_notify_topic,
             leaf_stack_sns_topic=self.sns_notify_topic,
             task_definition=self.container_nested_stack.task_definition,
-            instance_type=config["InstanceType"],
+            config=config["Ec2"],
             sg_container_traffic=self.sg_nested_stack.sg_container_traffic,
             efs_file_system=self.efs_nested_stack.efs_file_system,
             host_access_point=self.efs_nested_stack.host_access_point,

ContainerManager/utils/config_loader.py

@@ -4,20 +4,8 @@
 # https://github.com/mkaranasou/pyaml_env
 from pyaml_env import parse_config
 
-####################
-## HELPER METHODS ##
-####################
-def check_missing(config: dict, required_vars: list) -> None:
-    missing_vars = [x for x in required_vars if x not in config]
-    if any(missing_vars):
-        raise RuntimeError(f"Missing environment vars: [{', '.join(missing_vars)}]")
 
-#######################
-## BASE CONFIG LOGIC ##
-#######################
-def load_base_config(path: str) -> dict:
-    # TODO: Parse
-    return parse_config(path)
+
 
 #######################
 ## LEAF CONFIG LOGIC ##
@@ -26,84 +14,17 @@ def load_base_config(path: str) -> dict:
 # TODO: Re-write all of this, maybe create ec2.port objects instead of passing the dicts around all over the place.
 #         (And maybe do with other dicts here too)
 
-def parse_docker_ports(docker_ports_config: list) -> None:
-    for port_info in docker_ports_config:
-        ### Each port takes the format of {Protocol: Port}:
-        if len(port_info) != 1:
-            raise ValueError(f"Each port should have only one key-value pair. Got: {port_info}")
-        protocol, port = list(port_info.items())[0]
-
-        # Check if the protocol is valid:
-        valid_protocols = ["TCP", "UDP"]
-        if protocol.upper() not in valid_protocols:
-            raise NotImplementedError(f"Protocol {protocol} is not supported. Only {valid_protocols} are supported for now.")
-
-        # Check if the port is valid:
-        if not isinstance(port, int):
-            raise ValueError(f"Port {port} should be an integer.")
 
 
-required_leaf_vars = [
-    "InstanceType",
-    "Container",
-]
 def load_leaf_config(path: str) -> dict:
     config = parse_config(path)
 
-    ## Check base-level keys in the config:
-    check_missing(config, required_leaf_vars)
-
-    ## Check Container.* level keys in the config:
-    required_container_vars = [
-        "Image",
-        "Ports",
-    ]
-    check_missing(config["Container"], required_container_vars)
-    parse_docker_ports(config["Container"]["Ports"])
-
-    ## Make the environment variables strings:
-    environment = config["Container"].get("Environment", {})
-    environment = {key: str(val) for key, val in environment.items()}
-    config["Container"]["Environment"] = environment
-
-    ######################
-    ### VOLUMES CONFIG ###
-    ######################
-    volume = config.get("Volume", {})
-    if "RemovalPolicy" in volume:
-        volume["RemovalPolicy"] = volume["RemovalPolicy"].upper()
-    volume["EnableBackups"] = volume.get("EnableBackups", True)
-
-    config["Volume"] = volume
 
     #######################
     ### WATCHDOG CONFIG ###
     #######################
     watchdog = config.get("Watchdog", {})
-    if "MinutesWithoutConnections" not in watchdog:
-        watchdog["MinutesWithoutConnections"] = 5
-    assert watchdog["MinutesWithoutConnections"] >= 2, "MinutesWithoutConnections must be at least 2."
-
-    if "Type" not in watchdog:
-        using_tcp = any([list(x.keys())[0] == "TCP" for x in config["Container"]["Ports"]])
-        using_udp = any([list(x.keys())[0] == "UDP" for x in config["Container"]["Ports"]])
-        if using_tcp and using_udp:
-            raise ValueError("Watchdog type not specified, and could not be inferred from container ports. (Add Watchdog.Type)")
-        elif using_tcp:
-            watchdog["Type"] = "TCP"
-        elif using_udp:
-            watchdog["Type"] = "UDP"
-        else:
-            raise ValueError("Watchdog type not specified, and could not be inferred from container ports. (Add Watchdog.Type)")
-
-    if watchdog["Type"] == "TCP":
-        if "TcpPort" not in watchdog:
-            if len(config["Container"]["Ports"]) != 1:
-                raise ValueError("Cannot infer TCP port from multiple ports. (Add Watchdog.TcpPort)")
-            watchdog["TcpPort"] = list(config["Container"]["Ports"][0].values())[0]
-
-    elif watchdog["Type"] == "UDP":
-        pass
+
 
     # Default changes depending on protocol used:
     if "Threshold" not in watchdog: