From 411697e6a1d898f37b08c871199ba535006f1386 Mon Sep 17 00:00:00 2001 From: Jonathan Wright Date: Sat, 16 Mar 2024 15:51:44 +0000 Subject: [PATCH 1/2] Add initial k3s configuration with testing clusters Add the configuration to deploy single-node k3s clusters along with the testing-01 and testing-02 nodes to deploy it on. --- README.md | 8 ++ Taskfile.yaml | 8 +- inventory.yaml | 33 +++++ plays/group_vars/kubernetes.yaml | 44 ++++++ plays/group_vars/remote.yaml | 4 - plays/group_vars/testing.yaml | 4 + .../node-01.s.cym-south-1.kub3.uk.yaml | 25 +++- .../testing-01.t.cym-south-1.kub3.uk.yaml | 17 +++ .../testing-02.t.cym-south-1.kub3.uk.yaml | 17 +++ plays/kubernetes.yaml | 9 ++ plays/testing.yaml | 10 ++ plays/update.yaml | 2 +- plays/upgrade.yaml | 2 +- roles/k3s/README.md | 30 ++++ roles/k3s/defaults/main.yaml | 34 +++++ roles/k3s/handlers/main.yaml | 22 +++ roles/k3s/meta/main.yaml | 18 +++ roles/k3s/tasks/main.yaml | 129 ++++++++++++++++++ roles/k3s/templates/agent.yaml.jinja | 10 ++ roles/k3s/templates/cluster.yaml.jinja | 24 ++++ roles/k3s/templates/config.yaml.jinja | 5 + roles/k3s/templates/general.yaml.jinja | 3 + roles/k3s/templates/k3s.service.jinja | 0 roles/k3s/templates/modules.conf.jinja | 5 + roles/k3s/templates/node.yaml.jinja | 20 +++ roles/k3s/templates/server.yaml.jinja | 21 +++ roles/k3s/templates/service.conf.jinja | 11 ++ roles/k3s/templates/sysctl.conf.jinja | 5 + roles/k3s/templates/unit.conf.jinja | 4 + roles/k3s/vars/main.yaml | 26 ++++ roles/libvirtd/defaults/main.yaml | 4 +- roles/systemd_networkd/defaults/main.yaml | 6 + 32 files changed, 547 insertions(+), 13 deletions(-) delete mode 100644 plays/group_vars/remote.yaml create mode 100644 plays/group_vars/testing.yaml create mode 100644 plays/host_vars/testing-01.t.cym-south-1.kub3.uk.yaml create mode 100644 plays/host_vars/testing-02.t.cym-south-1.kub3.uk.yaml create mode 100644 plays/kubernetes.yaml create mode 100644 plays/testing.yaml create mode 100644 roles/k3s/README.md create mode 100644 roles/k3s/defaults/main.yaml create mode 100644 roles/k3s/handlers/main.yaml create mode 100644 roles/k3s/meta/main.yaml create mode 100644 roles/k3s/tasks/main.yaml create mode 100644 roles/k3s/templates/agent.yaml.jinja create mode 100644 roles/k3s/templates/cluster.yaml.jinja create mode 100644 roles/k3s/templates/config.yaml.jinja create mode 100644 roles/k3s/templates/general.yaml.jinja create mode 100644 roles/k3s/templates/k3s.service.jinja create mode 100644 roles/k3s/templates/modules.conf.jinja create mode 100644 roles/k3s/templates/node.yaml.jinja create mode 100644 roles/k3s/templates/server.yaml.jinja create mode 100644 roles/k3s/templates/service.conf.jinja create mode 100644 roles/k3s/templates/sysctl.conf.jinja create mode 100644 roles/k3s/templates/unit.conf.jinja create mode 100644 roles/k3s/vars/main.yaml diff --git a/README.md b/README.md index 7b09912..01db105 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ n3tuk. | [`libvirtd.yaml`][play-libvirtd] | [`libvirtd`][taskfile] | A play which will update the configuration of `libvirtd` on a system and prepare the Storage Pools. | | [`cache.yaml`][play-cache] | [`cache`][taskfile] | A play which will update the configuration of caching proxies. | | [`netdata.yaml`][play-netdata] | [`netdata`][taskfile] | A play which will update the configuration of Netdata on both parent and child nodes. | +| [`vault.yaml`][play-vault] | [`vault`][taskfile] | A play which will update the configuration of Hashicorp Vault in each of the environments of the Lab. | +| [`testing.yaml`][play-testing] | [`testing`][taskfile] | A play which will update the configuration of test Kubernetes clusters using k3s. | All Ansible plays run via `task` can be configured with limit overrides using `limit=` appended after the task: @@ -33,6 +35,8 @@ task: [bootstrap] ansible-playbook \ [play-libvirtd]: https://github.com/n3tuk/ansible/blob/main/plays/libvirtd.yaml [play-cache]: https://github.com/n3tuk/ansible/blob/main/plays/cache.yaml [play-netdata]: https://github.com/n3tuk/ansible/blob/main/plays/netdata.yaml +[play-vault]: https://github.com/n3tuk/ansible/blob/main/plays/vault.yaml +[play-testing]: https://github.com/n3tuk/ansible/blob/main/plays/testing.yaml [taskfile]: https://github.com/n3tuk/ansible/blob/main/Taskfile.yaml [inventory]: https://github.com/n3tuk/ansible/blob/main/inventory.yaml @@ -62,6 +66,8 @@ task: [bootstrap] ansible-playbook \ | [`cache`][role-cache] | A role to configure a caching proxy virtual host in nginx which will proxy and cache Arch Linux repositories and packages. | | [`logrotate`][role-logrotate] | A role to configure logrotate with sensible defaults to support the rotation and compression of historical log files. | | [`netdata`][role-netdata] | A role to configure netdata either as a parent node for centralised storage and processing, or a child to collect data and stream it to a parent node. | +| [`vault`][role-vault] | A role to install and configure Hashicorp Vault along with associated proxies, certificates, and firewall rules. | +| [`k3s`][role-k3s] | A role to install and configure k3s nodes to build and run Kubernetes clusters in different setups, including single-node as well as server/agent setups. | [role-filesystems]: https://github.com/n3tuk/ansible/tree/main/roles/filesystems [role-bootstrap]: https://github.com/n3tuk/ansible/tree/main/roles/bootstrap @@ -85,3 +91,5 @@ task: [bootstrap] ansible-playbook \ [role-cache]: https://github.com/n3tuk/ansible/tree/main/roles/cache [role-logrotate]: https://github.com/n3tuk/ansible/tree/main/roles/logrotate [role-netdata]: https://github.com/n3tuk/ansible/tree/main/roles/netdata +[role-vault]: https://github.com/n3tuk/ansible/tree/main/roles/vault +[role-k3s]: https://github.com/n3tuk/ansible/tree/main/roles/k3s diff --git a/Taskfile.yaml b/Taskfile.yaml index 3978c33..dfa05b1 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -219,16 +219,16 @@ tasks: --limit {{ .limit }} \ --forks 10 - cache: - desc: Deploy updated caching configurations + kubernetes: + desc: Deploy configurations for Kubernetes clusters silent: true cmds: - cmd: |- ansible-playbook \ - --syntax-check plays/cache.yaml + --syntax-check plays/kubernetes.yaml - cmd: |- ansible-playbook \ - --ask-become-pass plays/cache.yaml \ + --ask-become-pass plays/kubernetes.yaml \ --limit {{ .limit }} \ --forks 3 diff --git a/inventory.yaml b/inventory.yaml index ae72dbe..513685b 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -20,16 +20,49 @@ all: netdata-01.d.cym-south-1.kub3.uk: vault-[01:03].d.cym-south-1.kub3.uk: + testing: + hosts: + testing-[01:02].t.cym-south-1.kub3.uk: + virtual: children: cache: netdata: vault: + kubernetes: physical: children: nodes: + kubernetes: + children: + testing: + + bootstrap: + children: + nodes: + netdata: + services: + vault: + kubernetes: + + cache: + hosts: + cache-01.s.cym-south-1.kub3.uk: + + netdata: + hosts: + netdata-01.p.cym-south-1.kub3.uk: + netdata-01.d.cym-south-1.kub3.uk: + netdata-01.s.cym-south-1.kub3.uk: + + nodes: + hosts: + node-[01:06].p.cym-south-1.kub3.uk: + node-01.d.cym-south-1.kub3.uk: + node-01.s.cym-south-1.kub3.uk: + vault: hosts: vault-[01:03].p.cym-south-1.kub3.uk: diff --git a/plays/group_vars/kubernetes.yaml b/plays/group_vars/kubernetes.yaml index ed97d53..16d366c 100644 --- a/plays/group_vars/kubernetes.yaml +++ b/plays/group_vars/kubernetes.yaml @@ -1 +1,45 @@ --- +# Kubernetes nodes cannot operate with swap enabled, so ensure the filesystems +# are reconfigured to exclude that list. +filesystems_logical_volumes: + - name: system + group: storage + path: / + size: 8G + fs_type: ext4 + fs_opts: -L SYSTEM + - name: journald + group: storage + path: /var/log/journal + size: 256M + fs_type: ext4 + fs_opts: -L JOURNALD + mount_opts: + - nosuid + - nodev + - noexec + - name: pacman + group: storage + path: /var/cache/pacman + size: 2G + fs_type: ext4 + fs_opts: -L PACMAN + mount_opts: + - nosuid + - nodev + - noexec + - name: home + group: storage + path: /home + size: 256M + fs_type: ext4 + fs_opts: -L HOME + mount_opts: + - nosuid + - nodev + - name: rancher + group: storage + path: /var/lib/rancher + size: 32G + fs_type: ext4 + fs_opts: -L RANCHER diff --git a/plays/group_vars/remote.yaml b/plays/group_vars/remote.yaml deleted file mode 100644 index 4b13dbc..0000000 --- a/plays/group_vars/remote.yaml +++ /dev/null @@ -1,4 +0,0 @@ ---- -env_name: mythic-beasts -env_location: cym-south-1 -env_colour: ebcb8b diff --git a/plays/group_vars/testing.yaml b/plays/group_vars/testing.yaml new file mode 100644 index 0000000..65e21bb --- /dev/null +++ b/plays/group_vars/testing.yaml @@ -0,0 +1,4 @@ +--- +env_name: testing +env_location: cym-south-1 +env_colour: bright-yellow diff --git a/plays/host_vars/node-01.s.cym-south-1.kub3.uk.yaml b/plays/host_vars/node-01.s.cym-south-1.kub3.uk.yaml index 872ac13..fdb382c 100644 --- a/plays/host_vars/node-01.s.cym-south-1.kub3.uk.yaml +++ b/plays/host_vars/node-01.s.cym-south-1.kub3.uk.yaml @@ -11,6 +11,7 @@ systemd_networkd_vlans: - 30 - 31 - 32 + - 39 machines_list: - name: vault-01 @@ -34,7 +35,7 @@ machines_list: current: 2 volumes: - dev: vda - size: 512G + size: 128G vlan: 31 - name: netdata-01 environment: services @@ -69,3 +70,25 @@ machines_list: - dev: vda size: 1T vlan: 31 + - name: testing-01 + environment: testing + location: cym-south-1 + purpose: k3s-testing-node + vcpu: 4 + memory: + current: 12 + volumes: + - dev: vda + size: 256G + vlan: 39 + - name: testing-02 + environment: testing + location: cym-south-1 + purpose: k3s-testing-node + vcpu: 4 + memory: + current: 12 + volumes: + - dev: vda + size: 256G + vlan: 39 diff --git a/plays/host_vars/testing-01.t.cym-south-1.kub3.uk.yaml b/plays/host_vars/testing-01.t.cym-south-1.kub3.uk.yaml new file mode 100644 index 0000000..7fb9e83 --- /dev/null +++ b/plays/host_vars/testing-01.t.cym-south-1.kub3.uk.yaml @@ -0,0 +1,17 @@ +--- +env_purpose: k3s-single-node + +systemd_networkd_ipv4_address: 172.23.39.2/24 +systemd_networkd_ipv4_gateway: 172.23.39.1 +systemd_networkd_ipv6_address: 2a02:8010:8006:3a39:65:a6ff:fee8:a87a/64 + +bird_interface: "br-*" + +k3s_region: cym-south-1 +k3s_zone: cym-south-1a + +k3s_cluster_init: true + +k3s_service_cidr: 172.24.0.0/22 +k3s_cluster_dns: 172.24.0.10 +k3s_cluster_cidr: 172.24.64.0/18 diff --git a/plays/host_vars/testing-02.t.cym-south-1.kub3.uk.yaml b/plays/host_vars/testing-02.t.cym-south-1.kub3.uk.yaml new file mode 100644 index 0000000..0f0d5ac --- /dev/null +++ b/plays/host_vars/testing-02.t.cym-south-1.kub3.uk.yaml @@ -0,0 +1,17 @@ +--- +env_purpose: k3s-single-node + +systemd_networkd_ipv4_address: 172.23.39.3/24 +systemd_networkd_ipv4_gateway: 172.23.39.1 +systemd_networkd_ipv6_address: 2a02:8010:8006:3a39:1e:a3ff:fef7:def2/64 + +bird_interface: "br-*" + +k3s_region: cym-south-1 +k3s_zone: cym-south-1a + +k3s_cluster_init: true + +k3s_service_cidr: 172.24.4.0/22 +k3s_cluster_dns: 172.24.4.10 +k3s_cluster_cidr: 172.24.128.0/18 diff --git a/plays/kubernetes.yaml b/plays/kubernetes.yaml new file mode 100644 index 0000000..905c165 --- /dev/null +++ b/plays/kubernetes.yaml @@ -0,0 +1,9 @@ +--- +# Configure the caching service for Arch Linux repositories + +- name: Configure hosts for a single-node k3s deployments + hosts: testing + become: true + become_user: root + roles: + - role: k3s diff --git a/plays/testing.yaml b/plays/testing.yaml new file mode 100644 index 0000000..4f7a5b0 --- /dev/null +++ b/plays/testing.yaml @@ -0,0 +1,10 @@ +--- +# Configure the caching service for Arch Linux repositories + +- name: Configure the host for a single-node Kubernetes deployment + hosts: minikube + become: true + become_user: root + roles: + # - role: bird + - role: k3s diff --git a/plays/update.yaml b/plays/update.yaml index 05c4b99..762352b 100644 --- a/plays/update.yaml +++ b/plays/update.yaml @@ -3,7 +3,7 @@ # the virtual hosts first and completely before updating the physical hosts. - name: Update all virtual machines - hosts: virtual + hosts: virtual:!remote become: true become_user: root roles: diff --git a/plays/upgrade.yaml b/plays/upgrade.yaml index 27b944d..5d7c69f 100644 --- a/plays/upgrade.yaml +++ b/plays/upgrade.yaml @@ -7,7 +7,7 @@ ansible.builtin.import_playbook: update.yaml - name: Upgrade all virtual machines - hosts: virtual + hosts: virtual:!remote become: true become_user: root tasks: diff --git a/roles/k3s/README.md b/roles/k3s/README.md new file mode 100644 index 0000000..a5f1b20 --- /dev/null +++ b/roles/k3s/README.md @@ -0,0 +1,30 @@ +# n3t.uk Kubernetes (k3s) Ansible Role + +An Ansible role for the host preparation for the deployment of a Kubernetes +cluster, in both single-node and multi-node configurations, alongside the +creation and mounting of filesystems and the installation and configuration of +required system packages. + +## Requirements + +None. + +## Role Variables + +None. + +## Dependencies + +None. + +## Example Playbook + +```yaml +--- +- name: Configure a Kubernetes node + hosts: all + become: true + become_user: root + roles: + - role: k3s +``` diff --git a/roles/k3s/defaults/main.yaml b/roles/k3s/defaults/main.yaml new file mode 100644 index 0000000..7378905 --- /dev/null +++ b/roles/k3s/defaults/main.yaml @@ -0,0 +1,34 @@ +--- +# defaults file for k3s + +k3s_node_type: server + +k3s_node_region: unknown-1 +k3s_node_zone: unknown-1a + +k3s_node_labels: [] +k3s_node_taints: [] + +k3s_node_name: "{{ ansible_hostname }}" +k3s_bind_address: "{{ ansible_default_ipv4.address }}" +k3s_https_listen_port: 6433 + +k3s_flannel_backend: vxlan +k3s_service_cidr: 172.23.8.0/21 +k3s_cluster_dns: 172.23.8.10 +k3s_cluster_cidr: 172.23.16.0/20 +k3s_egress_selector_mode: agent + +k3s_cluster_init: false +k3s_tls_san: + - "{{ ansible_fqdn }}" + +k3s_disable_helm_controller: true +k3s_disable_network_policy: true +k3s_disable_embedded_registry: true + +k3s_service_disable: + - servicelb + - traefik + - local-storage + - metrics-server diff --git a/roles/k3s/handlers/main.yaml b/roles/k3s/handlers/main.yaml new file mode 100644 index 0000000..eed9d5f --- /dev/null +++ b/roles/k3s/handlers/main.yaml @@ -0,0 +1,22 @@ +--- +# handlers file for k3s + +- name: Reload k3s + ansible.builtin.systemd: + name: k3s.service + daemon_reload: false + state: restarted + +- name: Reload kernel modules + # noqa no-changed-when + ansible.builtin.command: | + modprobe "{{ module }}" + loop: "{{ k3s_kernel_modules }}" + loop_control: + label: "{{ module }}" + loop_var: module + +- name: Reload sysctl settings + # noqa no-changed-when + ansible.builtin.command: | + sysctl --system diff --git a/roles/k3s/meta/main.yaml b/roles/k3s/meta/main.yaml new file mode 100644 index 0000000..bed2e21 --- /dev/null +++ b/roles/k3s/meta/main.yaml @@ -0,0 +1,18 @@ +--- +galaxy_info: + author: Jonathan Wright + description: Set up and prepare a single-node Kubernetes deployment + company: n3tuk + issue_tracker_url: https://github.com/n3tuk/ansible/issues + license: MIT + min_ansible_version: "2.15" + platforms: + - name: ArchLinux + versions: + - all + + galaxy_tags: + - k3s + - kubernetes + +dependencies: [] diff --git a/roles/k3s/tasks/main.yaml b/roles/k3s/tasks/main.yaml new file mode 100644 index 0000000..eb08645 --- /dev/null +++ b/roles/k3s/tasks/main.yaml @@ -0,0 +1,129 @@ +--- +# tasks file for k3s + +- name: Check this role is not being run under bootstrap conditions + ansible.builtin.fail: + msg: This role does not support being run when bootstrapping hosts. Failing. + when: bootstrap_mount_base | default('') | length + +- name: Configure required kernel modules + ansible.builtin.template: + src: modules.conf.jinja + dest: /etc/modules-load.d/80-k3s.conf + owner: root + group: root + mode: u=rw,g=r,o=r + notify: + - Reload kernel modules + tags: + - k3s + - modules + - configuraton + +- name: Configure sysctl settings + ansible.builtin.template: + src: sysctl.conf.jinja + dest: /etc/sysctl.d/80-k3s.conf + owner: root + group: root + mode: u=rw,g=r,o=r + notify: + - Reload sysctl settings + tags: + - k3s + - sysctl + - configuraton + +- name: Install k3s pre-requisites + community.general.pacman: + name: "{{ k3s_packages }}" + state: latest + force: true + extra_args: --noconfirm + tags: + - k3s + - package + +- name: Ensure required k3s services are enabled + ansible.builtin.systemd_service: + name: "{{ service }}" + enabled: true + state: started + loop: "{{ k3s_services }}" + loop_control: + label: "{{ service }}" + loop_var: service + tags: + - k3s + - service + +- name: Prepare the k3s directories + ansible.builtin.file: + path: "{{ directory }}" + state: directory + owner: root + group: root + mode: u=rwx,g=rx,o=rx + loop: + - /etc/rancher/k3s/config.yaml.d + - /etc/systemd/system/k3s.service.d + loop_control: + loop_var: directory + label: "{{ directory }}" + tags: + - k3s + - configuration + +- name: Install the k3s.service overrides + ansible.builtin.template: + src: "{{ override }}.jinja" + dest: "/etc/systemd/system/k3s.service.d/{{ override }}" + owner: root + group: root + mode: u=rw,g=r,o=r + loop: + - service.conf + - unit.conf + loop_control: + loop_var: override + label: "/etc/systemd/system/k3s.service.d/{{ override }}" + notify: + - Reload k3s + tags: + - k3s + - service + +- name: Install the general k3s configuration + ansible.builtin.template: + src: config.yaml.jinja + dest: /etc/rancher/k3s/config.yaml + owner: root + group: root + mode: u=rw,g=r,o=r + notify: + - Reload k3s + tags: + - k3s + - configuraton + +- name: Install the drop-in k3s configurations + ansible.builtin.template: + src: "{{ config }}.jinja" + dest: "/etc/rancher/k3s/config.yaml.d/{{ config }}" + owner: root + group: root + mode: u=rw,g=r,o=r + when: k3s_node_type == "server" or config != "server.yaml" + loop: + - server.yaml + - agent.yaml + - cluster.yaml + - node.yaml + loop_control: + loop_var: config + label: "/etc/rancher/k3s/config.yaml.d/{{ config }}" + notify: + - Reload k3s + tags: + - k3s + - configuraton diff --git a/roles/k3s/templates/agent.yaml.jinja b/roles/k3s/templates/agent.yaml.jinja new file mode 100644 index 0000000..d0b3c88 --- /dev/null +++ b/roles/k3s/templates/agent.yaml.jinja @@ -0,0 +1,10 @@ +--- +# {{ ansible_managed }} +# k3s Agent Configuration +{% if k3s_node_type == "agent" + and k3s_agent_server is defined + and k3s_agent_token is defined %} + +server: {{ k3s_agent_server }} +token: {{ k3s_agent_token }} +{% endif %} diff --git a/roles/k3s/templates/cluster.yaml.jinja b/roles/k3s/templates/cluster.yaml.jinja new file mode 100644 index 0000000..b873f5b --- /dev/null +++ b/roles/k3s/templates/cluster.yaml.jinja @@ -0,0 +1,24 @@ +--- +# {{ ansible_managed }} +# k3s Cluster Configuration + +secrets-encryption: true + +flannel-backend: {{ k3s_flannel_backend }} +service-cidr: {{ k3s_service_cidr }} +cluster-cidr: {{ k3s_cluster_cidr }} +cluster-dns: {{ k3s_cluster_dns }} + +egress-selector-mode: {{ k3s_egress_selector_mode }} + +embedded-registry: {{ 'false' if k3s_disable_embedded_registry else 'true' }} + +disable-helm-controller: {{ 'true' if k3s_disable_helm_controller else 'false' }} +disable-network-policy: {{ 'true' if k3s_disable_network_policy else 'false' }} +{% if k3s_service_disable | length > 0 %} + +disable: +{% for service in k3s_service_disable %} + - {{ service }} +{% endfor %} +{% endif %} diff --git a/roles/k3s/templates/config.yaml.jinja b/roles/k3s/templates/config.yaml.jinja new file mode 100644 index 0000000..1c37fbc --- /dev/null +++ b/roles/k3s/templates/config.yaml.jinja @@ -0,0 +1,5 @@ +--- +# {{ ansible_managed }} +# k3s General Configuration + +write-kubeconfig-mode: 600 diff --git a/roles/k3s/templates/general.yaml.jinja b/roles/k3s/templates/general.yaml.jinja new file mode 100644 index 0000000..483c562 --- /dev/null +++ b/roles/k3s/templates/general.yaml.jinja @@ -0,0 +1,3 @@ +--- +# {{ ansible_managed }} +write-kubeconfig-mode: 600 diff --git a/roles/k3s/templates/k3s.service.jinja b/roles/k3s/templates/k3s.service.jinja new file mode 100644 index 0000000..e69de29 diff --git a/roles/k3s/templates/modules.conf.jinja b/roles/k3s/templates/modules.conf.jinja new file mode 100644 index 0000000..7a5f811 --- /dev/null +++ b/roles/k3s/templates/modules.conf.jinja @@ -0,0 +1,5 @@ +# {{ ansible_managed }} + +{% for module in k3s_kernel_modules %} +{{ module }} +{% endfor %} diff --git a/roles/k3s/templates/node.yaml.jinja b/roles/k3s/templates/node.yaml.jinja new file mode 100644 index 0000000..4787df6 --- /dev/null +++ b/roles/k3s/templates/node.yaml.jinja @@ -0,0 +1,20 @@ +--- +# {{ ansible_managed }} +# k3s Node Configuration + +node-name: {{ k3s_node_name }} +bind-address: {{ k3s_bind_address }} +{% if k3s_node_taints|length > 0 %} + +node-taint+: +{% for taint in k3s_node_taints %} + - {{ taint.key }}={{ taint.value }} +{% endfor %} +{% endif %} + +node-label+: + - topology.kubernetes.io/region={{ k3s_node_region }} + - topology.kubernetes.io/zone={{ k3s_node_zone }} +{% for label in k3s_node_labels %} + - {{ label.key }}={{ taint.value }} +{% endfor %} diff --git a/roles/k3s/templates/server.yaml.jinja b/roles/k3s/templates/server.yaml.jinja new file mode 100644 index 0000000..bb7916d --- /dev/null +++ b/roles/k3s/templates/server.yaml.jinja @@ -0,0 +1,21 @@ +--- +# {{ ansible_managed }} +# k3s Server Configuration +{% if k3s_cluster_init %} + +cluster-init: true +{% if k3s_tls_san | length > 0 %} +tls-san: +{% for san in k3s_tls_san %} + - {{ san }} +{% endfor %} +{% endif %} +{% elif k3s_node_type == "agent" + and k3s_agent_server is defined + and k3s_agent_token is defined %} + +server: {{ k3s_agent_server }} +token: {{ k3s_agent_token }} +{% endif %} + +https-listen-port: {{ k3s_https_listen_port }} diff --git a/roles/k3s/templates/service.conf.jinja b/roles/k3s/templates/service.conf.jinja new file mode 100644 index 0000000..66ee061 --- /dev/null +++ b/roles/k3s/templates/service.conf.jinja @@ -0,0 +1,11 @@ +# {{ ansible_managed }} + +[Service] +# The following empty lines are needed to clear the previous entries for these +# values set in the original k3s.service from the upstream package +EnvironmentFile= +ExecStartPre= +ExecStart= + +# Override the k3s call depending on if we want to run the server or the agent +ExecStart=/usr/bin/k3s {{ 'server' if k3s_node_type == 'server' else 'agent' }} diff --git a/roles/k3s/templates/sysctl.conf.jinja b/roles/k3s/templates/sysctl.conf.jinja new file mode 100644 index 0000000..14c825c --- /dev/null +++ b/roles/k3s/templates/sysctl.conf.jinja @@ -0,0 +1,5 @@ +# {{ ansible_managed }} + +{% for entry in k3s_sysctl_settings %} +{{ entry.key }} = {{ entry.value }} +{% endfor %} diff --git a/roles/k3s/templates/unit.conf.jinja b/roles/k3s/templates/unit.conf.jinja new file mode 100644 index 0000000..942c785 --- /dev/null +++ b/roles/k3s/templates/unit.conf.jinja @@ -0,0 +1,4 @@ +# {{ ansible_managed }} + +[Unit] +Description=Lightweight Kubernetes {{ k3s_node_type | title }} diff --git a/roles/k3s/vars/main.yaml b/roles/k3s/vars/main.yaml new file mode 100644 index 0000000..85696c8 --- /dev/null +++ b/roles/k3s/vars/main.yaml @@ -0,0 +1,26 @@ +--- +# vars file for k3s + +k3s_packages: + - conntrack-tools + - k3s-1.29-bin + - cni-plugins + - containerd + - nerdctl + - kubectl + +k3s_services: + - conntrackd.service + - containerd.service + +k3s_kernel_modules: + - overlay + - br_netfilter + +k3s_sysctl_settings: + - key: net.bridge.bridge-nf-call-iptables + value: 1 + - key: net.bridge.bridge-nf-call-ip6tables + value: 1 + - key: net.ipv4.ip_forward + value: 1 diff --git a/roles/libvirtd/defaults/main.yaml b/roles/libvirtd/defaults/main.yaml index 8543cfe..318edae 100644 --- a/roles/libvirtd/defaults/main.yaml +++ b/roles/libvirtd/defaults/main.yaml @@ -6,8 +6,8 @@ libvirtd_snapshots_volume_group: storage libvirtd_snapshots_volume_size: 64G libvirtd_image_arch_prefix: https://mirror.bytemark.co.uk/archlinux/iso/ -libvirtd_image_arch_checksum: 477f50617d648e46d6e326549aa56ab92115a29a97f2ca364e944cea06970608 -libvirtd_image_arch_version: 2023.11.01 +libvirtd_image_arch_checksum: 891ebab4661cedb0ae3b8fe15a906ae2ba22e284551dc293436d5247220933c5 +libvirtd_image_arch_version: 2024.02.01 libvirtd_volume_groups: - storage diff --git a/roles/systemd_networkd/defaults/main.yaml b/roles/systemd_networkd/defaults/main.yaml index be2989f..03e8921 100644 --- a/roles/systemd_networkd/defaults/main.yaml +++ b/roles/systemd_networkd/defaults/main.yaml @@ -11,3 +11,9 @@ systemd_networkd_ipv4_address: systemd_networkd_ipv4_gateway: systemd_networkd_ipv6_address: + +systemd_networkd_dummy_interfaces: [] +# - enabled: false +# name: lb01 +# ipv4: 172.23.0.1/32 +# ipv6: 2a02:8010:8006:3a00::1/128 From b9fc19dc6a167880effd40e5e8efe3b69bc21662 Mon Sep 17 00:00:00 2001 From: Jonathan Wright Date: Sun, 18 Feb 2024 11:32:30 +0000 Subject: [PATCH 2/2] Allow customisation of Netdata Parent for streaming Allow the target for the Netdata Parent to be changed, if required, so another Parent can be used for the nodes to stream to. --- .../netdata-01.d.cym-south-1.kub3.uk.yaml | 8 +++++++- .../netdata-01.p.cym-south-1.kub3.uk.yaml | 8 +++++++- .../netdata-01.s.cym-south-1.kub3.uk.yaml | 8 +++++++- roles/netdata/defaults/main.yaml | 8 ++++++++ .../netdata/templates/child.stream.conf.jinja | 2 +- .../templates/parent.netdata.conf.jinja | 18 ++++++------------ roles/netdata/vars/main.yaml | 1 - 7 files changed, 36 insertions(+), 17 deletions(-) diff --git a/plays/host_vars/netdata-01.d.cym-south-1.kub3.uk.yaml b/plays/host_vars/netdata-01.d.cym-south-1.kub3.uk.yaml index 2015d06..d197cf2 100644 --- a/plays/host_vars/netdata-01.d.cym-south-1.kub3.uk.yaml +++ b/plays/host_vars/netdata-01.d.cym-south-1.kub3.uk.yaml @@ -5,4 +5,10 @@ systemd_networkd_ipv4_address: 172.23.31.6/24 systemd_networkd_ipv4_gateway: 172.23.31.1 systemd_networkd_ipv6_address: 2a02:8010:8006:3a31:a6:4eff:fe95:52c8/64 -netdata_cache_volume_size: 475G +netdata_volume_size: 475G +netdata_dbengine_teir0_space: 200000 +netdata_dbengine_teir0_cache: 512 +netdata_dbengine_teir1_space: 150000 +netdata_dbengine_teir1_cache: 128 +netdata_dbengine_teir2_space: 100000 +netdata_dbengine_teir2_cache: 128 diff --git a/plays/host_vars/netdata-01.p.cym-south-1.kub3.uk.yaml b/plays/host_vars/netdata-01.p.cym-south-1.kub3.uk.yaml index 48d858b..17d921a 100644 --- a/plays/host_vars/netdata-01.p.cym-south-1.kub3.uk.yaml +++ b/plays/host_vars/netdata-01.p.cym-south-1.kub3.uk.yaml @@ -5,4 +5,10 @@ systemd_networkd_ipv4_address: 172.23.31.5/24 systemd_networkd_ipv4_gateway: 172.23.31.1 systemd_networkd_ipv6_address: 2a02:8010:8006:3a31:a4:83ff:fece:564b/64 -netdata_cache_volume_size: 1000G +netdata_volume_size: 1000G +netdata_dbengine_teir0_space: 500000 +netdata_dbengine_teir0_cache: 1024 +netdata_dbengine_teir1_space: 250000 +netdata_dbengine_teir1_cache: 512 +netdata_dbengine_teir2_space: 200000 +netdata_dbengine_teir2_cache: 512 diff --git a/plays/host_vars/netdata-01.s.cym-south-1.kub3.uk.yaml b/plays/host_vars/netdata-01.s.cym-south-1.kub3.uk.yaml index d0c56c2..96d8848 100644 --- a/plays/host_vars/netdata-01.s.cym-south-1.kub3.uk.yaml +++ b/plays/host_vars/netdata-01.s.cym-south-1.kub3.uk.yaml @@ -5,4 +5,10 @@ systemd_networkd_ipv4_address: 172.23.31.7/24 systemd_networkd_ipv4_gateway: 172.23.31.1 systemd_networkd_ipv6_address: 2a02:8010:8006:3a31:90:d4ff:feae:df72/64 -netdata_cache_volume_size: 225G +netdata_volume_size: 225G +netdata_dbengine_teir0_space: 125000 +netdata_dbengine_teir0_cache: 512 +netdata_dbengine_teir1_space: 50000 +netdata_dbengine_teir1_cache: 128 +netdata_dbengine_teir2_space: 40000 +netdata_dbengine_teir2_cache: 128 diff --git a/roles/netdata/defaults/main.yaml b/roles/netdata/defaults/main.yaml index 4dd4f48..2a14f2c 100644 --- a/roles/netdata/defaults/main.yaml +++ b/roles/netdata/defaults/main.yaml @@ -7,6 +7,7 @@ netdata_node_type: child netdata_stream_port: 19999 netdata_stream_encrypted: false +netdata_stream_destination: "{{ env_name }}" netdata_cloud_claim_token: netdata_cloud_claim_room: @@ -14,6 +15,13 @@ netdata_cloud_claim_url: https://app.netdata.cloud netdata_volume_size: 64G +netdata_dbengine_teir0_space: 10240 +netdata_dbengine_teir0_cache: 512 +netdata_dbengine_teir1_space: 5120 +netdata_dbengine_teir1_cache: 32 +netdata_dbengine_teir2_space: 4096 +netdata_dbengine_teir2_cache: 32 + netdata_alarm_pushover_enabled: true netdata_alarm_pushover_app_token: netdata_alarm_pushover_recipient: diff --git a/roles/netdata/templates/child.stream.conf.jinja b/roles/netdata/templates/child.stream.conf.jinja index ca94cc2..82f17fc 100644 --- a/roles/netdata/templates/child.stream.conf.jinja +++ b/roles/netdata/templates/child.stream.conf.jinja @@ -7,7 +7,7 @@ # Send all available metrics from this client node to a parent node as the # destination for storage and processing of alerts enabled = yes - destination = tcp:netdata-01.{{ env_name[:1] }}.{{ env_location }}.kub3.uk:{{ netdata_stream_port }}{% if netdata_stream_encrypted %}:SSL{% endif %} + destination = tcp:netdata-01.{{ netdata_stream_destination[:1] }}.{{ env_location }}.kub3.uk:{{ netdata_stream_port }}{% if netdata_stream_encrypted %}:SSL{% endif %} api key = {{ netdata_api_key }} send charts matching = * diff --git a/roles/netdata/templates/parent.netdata.conf.jinja b/roles/netdata/templates/parent.netdata.conf.jinja index e48d2aa..62e28bc 100644 --- a/roles/netdata/templates/parent.netdata.conf.jinja +++ b/roles/netdata/templates/parent.netdata.conf.jinja @@ -15,23 +15,17 @@ # Storage Tier 0 update every = 1 # 1 Second - # Provide 14 days of 1s metrics - # (9500 x 1byte x 86400 seconds) = 765MB per day - dbengine multihost disk space MB = 10240 - dbengine page cache size MB = 512 + dbengine multihost disk space MB = {{ netdata_dbengine_teir0_space }} + dbengine page cache size MB = {{ netdata_dbengine_teir0_cache }} # Storage Tier 1 dbengine tier 1 update every iterations = 60 # 60 Seconds - # Provide 90 days of 1m metrics - # (9500 x 4byte x 1440 minutes) = 50MB per day - dbengine tier 1 multihost disk space MB = 5120 - dbengine tier 1 page cache size MB = 32 + dbengine tier 1 multihost disk space MB = {{ netdata_dbengine_teir1_space }} + dbengine tier 1 page cache size MB = {{ netdata_dbengine_teir1_cache }} dbengine tier 1 backfill = new # Storage Tier 2 dbengine tier 2 update every iterations = 60 # 3600 Seconds - # Provide 365 days of 5m metrics - # (9500 x 4byte x 288 x 5 minutes) = 10MB per day - dbengine tier 2 multihost disk space MB = 4096 - dbengine tier 2 page cache size MB = 32 + dbengine tier 2 multihost disk space MB = {{ netdata_dbengine_teir2_space }} + dbengine tier 2 page cache size MB = {{ netdata_dbengine_teir2_cache }} dbengine tier 2 backfill = new diff --git a/roles/netdata/vars/main.yaml b/roles/netdata/vars/main.yaml index 0493ad2..df5934e 100644 --- a/roles/netdata/vars/main.yaml +++ b/roles/netdata/vars/main.yaml @@ -3,7 +3,6 @@ netdata_packages: - netdata - - netdata-go-plugins # Needed for access to the hostname binary - inetutils