Merge pull request #15 from n3tuk/add-k3s-testing

Add initial k3s configuration with `testing` clusters

README.md

@@ -15,6 +15,8 @@ n3tuk.
 | [`libvirtd.yaml`][play-libvirtd]   | [`libvirtd`][taskfile]  | A play which will update the configuration of `libvirtd` on a system and prepare the Storage Pools.                                              |
 | [`cache.yaml`][play-cache]         | [`cache`][taskfile]     | A play which will update the configuration of caching proxies.                                                                                   |
 | [`netdata.yaml`][play-netdata]     | [`netdata`][taskfile]   | A play which will update the configuration of Netdata on both parent and child nodes.                                                            |
+| [`vault.yaml`][play-vault]         | [`vault`][taskfile]     | A play which will update the configuration of Hashicorp Vault in each of the environments of the Lab.                                            |
+| [`testing.yaml`][play-testing]     | [`testing`][taskfile]   | A play which will update the configuration of test Kubernetes clusters using k3s.                                                                |
 
 All Ansible plays run via `task` can be configured with limit overrides using
 `limit=` appended after the task:
@@ -33,6 +35,8 @@ task: [bootstrap] ansible-playbook \
 [play-libvirtd]: https://github.com/n3tuk/ansible/blob/main/plays/libvirtd.yaml
 [play-cache]: https://github.com/n3tuk/ansible/blob/main/plays/cache.yaml
 [play-netdata]: https://github.com/n3tuk/ansible/blob/main/plays/netdata.yaml
+[play-vault]: https://github.com/n3tuk/ansible/blob/main/plays/vault.yaml
+[play-testing]: https://github.com/n3tuk/ansible/blob/main/plays/testing.yaml
 [taskfile]: https://github.com/n3tuk/ansible/blob/main/Taskfile.yaml
 [inventory]: https://github.com/n3tuk/ansible/blob/main/inventory.yaml
 
@@ -62,6 +66,8 @@ task: [bootstrap] ansible-playbook \
 | [`cache`][role-cache]                       | A role to configure a caching proxy virtual host in nginx which will proxy and cache Arch Linux repositories and packages.                                                                                    |
 | [`logrotate`][role-logrotate]               | A role to configure logrotate with sensible defaults to support the rotation and compression of historical log files.                                                                                         |
 | [`netdata`][role-netdata]                   | A role to configure netdata either as a parent node for centralised storage and processing, or a child to collect data and stream it to a parent node.                                                        |
+| [`vault`][role-vault]                       | A role to install and configure Hashicorp Vault along with associated proxies, certificates, and firewall rules.                                                                                              |
+| [`k3s`][role-k3s]                           | A role to install and configure k3s nodes to build and run Kubernetes clusters in different setups, including single-node as well as server/agent setups.                                                     |
 
 [role-filesystems]: https://github.com/n3tuk/ansible/tree/main/roles/filesystems
 [role-bootstrap]: https://github.com/n3tuk/ansible/tree/main/roles/bootstrap
@@ -85,3 +91,5 @@ task: [bootstrap] ansible-playbook \
 [role-cache]: https://github.com/n3tuk/ansible/tree/main/roles/cache
 [role-logrotate]: https://github.com/n3tuk/ansible/tree/main/roles/logrotate
 [role-netdata]: https://github.com/n3tuk/ansible/tree/main/roles/netdata
+[role-vault]: https://github.com/n3tuk/ansible/tree/main/roles/vault
+[role-k3s]: https://github.com/n3tuk/ansible/tree/main/roles/k3s

Taskfile.yaml

@@ -219,16 +219,16 @@ tasks:
             --limit {{ .limit }} \
             --forks 10
 
-  cache:
-    desc: Deploy updated caching configurations
+  kubernetes:
+    desc: Deploy configurations for Kubernetes clusters
     silent: true
     cmds:
       - cmd: |-
           ansible-playbook \
-            --syntax-check plays/cache.yaml
+            --syntax-check plays/kubernetes.yaml
       - cmd: |-
           ansible-playbook \
-            --ask-become-pass plays/cache.yaml \
+            --ask-become-pass plays/kubernetes.yaml \
             --limit {{ .limit }} \
             --forks 3
 

inventory.yaml

@@ -20,16 +20,49 @@ all:
         netdata-01.d.cym-south-1.kub3.uk:
         vault-[01:03].d.cym-south-1.kub3.uk:
 
+    testing:
+      hosts:
+        testing-[01:02].t.cym-south-1.kub3.uk:
+
     virtual:
       children:
         cache:
         netdata:
         vault:
+        kubernetes:
 
     physical:
       children:
         nodes:
 
+    kubernetes:
+      children:
+        testing:
+
+    bootstrap:
+      children:
+        nodes:
+        netdata:
+        services:
+        vault:
+        kubernetes:
+
+    cache:
+      hosts:
+        cache-01.s.cym-south-1.kub3.uk:
+
+    netdata:
+      hosts:
+        netdata-01.p.cym-south-1.kub3.uk:
+        netdata-01.d.cym-south-1.kub3.uk:
+        netdata-01.s.cym-south-1.kub3.uk:
+
+    nodes:
+      hosts:
+        node-[01:06].p.cym-south-1.kub3.uk:
+        node-01.d.cym-south-1.kub3.uk:
+        node-01.s.cym-south-1.kub3.uk:
+
     vault:
       hosts:
         vault-[01:03].p.cym-south-1.kub3.uk:

plays/group_vars/kubernetes.yaml

@@ -1 +1,45 @@
 ---
+# Kubernetes nodes cannot operate with swap enabled, so ensure the filesystems
+# are reconfigured to exclude that list.
+filesystems_logical_volumes:
+  - name: system
+    group: storage
+    path: /
+    size: 8G
+    fs_type: ext4
+    fs_opts: -L SYSTEM
+  - name: journald
+    group: storage
+    path: /var/log/journal
+    size: 256M
+    fs_type: ext4
+    fs_opts: -L JOURNALD
+    mount_opts:
+      - nosuid
+      - nodev
+      - noexec
+  - name: pacman
+    group: storage
+    path: /var/cache/pacman
+    size: 2G
+    fs_type: ext4
+    fs_opts: -L PACMAN
+    mount_opts:
+      - nosuid
+      - nodev
+      - noexec
+  - name: home
+    group: storage
+    path: /home
+    size: 256M
+    fs_type: ext4
+    fs_opts: -L HOME
+    mount_opts:
+      - nosuid
+      - nodev
+  - name: rancher
+    group: storage
+    path: /var/lib/rancher
+    size: 32G
+    fs_type: ext4
+    fs_opts: -L RANCHER

plays/group_vars/testing.yaml

@@ -0,0 +1,4 @@
+---
+env_name: testing
+env_location: cym-south-1
+env_colour: bright-yellow

plays/host_vars/netdata-01.d.cym-south-1.kub3.uk.yaml

@@ -5,4 +5,10 @@ systemd_networkd_ipv4_address: 172.23.31.6/24
 systemd_networkd_ipv4_gateway: 172.23.31.1
 systemd_networkd_ipv6_address: 2a02:8010:8006:3a31:a6:4eff:fe95:52c8/64
 
-netdata_cache_volume_size: 475G
+netdata_volume_size: 475G
+netdata_dbengine_teir0_space: 200000
+netdata_dbengine_teir0_cache: 512
+netdata_dbengine_teir1_space: 150000
+netdata_dbengine_teir1_cache: 128
+netdata_dbengine_teir2_space: 100000
+netdata_dbengine_teir2_cache: 128

plays/host_vars/netdata-01.p.cym-south-1.kub3.uk.yaml

@@ -5,4 +5,10 @@ systemd_networkd_ipv4_address: 172.23.31.5/24
 systemd_networkd_ipv4_gateway: 172.23.31.1
 systemd_networkd_ipv6_address: 2a02:8010:8006:3a31:a4:83ff:fece:564b/64
 
-netdata_cache_volume_size: 1000G
+netdata_volume_size: 1000G
+netdata_dbengine_teir0_space: 500000
+netdata_dbengine_teir0_cache: 1024
+netdata_dbengine_teir1_space: 250000
+netdata_dbengine_teir1_cache: 512
+netdata_dbengine_teir2_space: 200000
+netdata_dbengine_teir2_cache: 512

plays/host_vars/netdata-01.s.cym-south-1.kub3.uk.yaml

@@ -5,4 +5,10 @@ systemd_networkd_ipv4_address: 172.23.31.7/24
 systemd_networkd_ipv4_gateway: 172.23.31.1
 systemd_networkd_ipv6_address: 2a02:8010:8006:3a31:90:d4ff:feae:df72/64
 
-netdata_cache_volume_size: 225G
+netdata_volume_size: 225G
+netdata_dbengine_teir0_space: 125000
+netdata_dbengine_teir0_cache: 512
+netdata_dbengine_teir1_space: 50000
+netdata_dbengine_teir1_cache: 128
+netdata_dbengine_teir2_space: 40000
+netdata_dbengine_teir2_cache: 128

plays/host_vars/node-01.s.cym-south-1.kub3.uk.yaml

@@ -11,6 +11,7 @@ systemd_networkd_vlans:
   - 30
   - 31
   - 32
+  - 39
 
 machines_list:
   - name: vault-01
@@ -34,7 +35,7 @@ machines_list:
       current: 2
     volumes:
       - dev: vda
-        size: 512G
+        size: 128G
     vlan: 31
   - name: netdata-01
     environment: services
@@ -69,3 +70,25 @@ machines_list:
       - dev: vda
         size: 1T
     vlan: 31
+  - name: testing-01
+    environment: testing
+    location: cym-south-1
+    purpose: k3s-testing-node
+    vcpu: 4
+    memory:
+      current: 12
+    volumes:
+      - dev: vda
+        size: 256G
+    vlan: 39
+  - name: testing-02
+    environment: testing
+    location: cym-south-1
+    purpose: k3s-testing-node
+    vcpu: 4
+    memory:
+      current: 12
+    volumes:
+      - dev: vda
+        size: 256G
+    vlan: 39

plays/host_vars/testing-01.t.cym-south-1.kub3.uk.yaml

@@ -0,0 +1,17 @@
+---
+env_purpose: k3s-single-node
+
+systemd_networkd_ipv4_address: 172.23.39.2/24
+systemd_networkd_ipv4_gateway: 172.23.39.1
+systemd_networkd_ipv6_address: 2a02:8010:8006:3a39:65:a6ff:fee8:a87a/64
+
+bird_interface: "br-*"
+
+k3s_region: cym-south-1
+k3s_zone: cym-south-1a
+
+k3s_cluster_init: true
+
+k3s_service_cidr: 172.24.0.0/22
+k3s_cluster_dns: 172.24.0.10
+k3s_cluster_cidr: 172.24.64.0/18

plays/host_vars/testing-02.t.cym-south-1.kub3.uk.yaml

@@ -0,0 +1,17 @@
+---
+env_purpose: k3s-single-node
+
+systemd_networkd_ipv4_address: 172.23.39.3/24
+systemd_networkd_ipv4_gateway: 172.23.39.1
+systemd_networkd_ipv6_address: 2a02:8010:8006:3a39:1e:a3ff:fef7:def2/64
+
+bird_interface: "br-*"
+
+k3s_region: cym-south-1
+k3s_zone: cym-south-1a
+
+k3s_cluster_init: true
+
+k3s_service_cidr: 172.24.4.0/22
+k3s_cluster_dns: 172.24.4.10
+k3s_cluster_cidr: 172.24.128.0/18

plays/kubernetes.yaml

@@ -0,0 +1,9 @@
+---
+# Configure the caching service for Arch Linux repositories
+
+- name: Configure hosts for a single-node k3s deployments
+  hosts: testing
+  become: true
+  become_user: root
+  roles:
+    - role: k3s

plays/testing.yaml

@@ -0,0 +1,10 @@
+---
+# Configure the caching service for Arch Linux repositories
+
+- name: Configure the host for a single-node Kubernetes deployment
+  hosts: minikube
+  become: true
+  become_user: root
+  roles:
+    # - role: bird
+    - role: k3s

plays/update.yaml

@@ -3,7 +3,7 @@
 # the virtual hosts first and completely before updating the physical hosts.
 
 - name: Update all virtual machines
-  hosts: virtual
+  hosts: virtual:!remote
   become: true
   become_user: root
   roles:

plays/upgrade.yaml

@@ -7,7 +7,7 @@
   ansible.builtin.import_playbook: update.yaml
 
 - name: Upgrade all virtual machines
-  hosts: virtual
+  hosts: virtual:!remote
   become: true
   become_user: root
   tasks:

roles/k3s/README.md

@@ -0,0 +1,30 @@
+# n3t.uk Kubernetes (k3s) Ansible Role
+
+An Ansible role for the host preparation for the deployment of a Kubernetes
+cluster, in both single-node and multi-node configurations, alongside the
+creation and mounting of filesystems and the installation and configuration of
+required system packages.
+
+## Requirements
+
+None.
+
+## Role Variables
+
+None.
+
+## Dependencies
+
+None.
+
+## Example Playbook
+
+```yaml
+---
+- name: Configure a Kubernetes node
+  hosts: all
+  become: true
+  become_user: root
+  roles:
+    - role: k3s
+```

roles/k3s/defaults/main.yaml

@@ -0,0 +1,34 @@
+---
+# defaults file for k3s
+
+k3s_node_type: server
+
+k3s_node_region: unknown-1
+k3s_node_zone: unknown-1a
+
+k3s_node_labels: []
+k3s_node_taints: []
+
+k3s_node_name: "{{ ansible_hostname }}"
+k3s_bind_address: "{{ ansible_default_ipv4.address }}"
+k3s_https_listen_port: 6433
+
+k3s_flannel_backend: vxlan
+k3s_service_cidr: 172.23.8.0/21
+k3s_cluster_dns: 172.23.8.10
+k3s_cluster_cidr: 172.23.16.0/20
+k3s_egress_selector_mode: agent
+
+k3s_cluster_init: false
+k3s_tls_san:
+  - "{{ ansible_fqdn }}"
+
+k3s_disable_helm_controller: true
+k3s_disable_network_policy: true
+k3s_disable_embedded_registry: true
+
+k3s_service_disable:
+  - servicelb
+  - traefik
+  - local-storage
+  - metrics-server