CI: switch from LXD to Lima

Alternative to PR 332

Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>

.github/workflows/main.yaml

@@ -59,32 +59,49 @@ jobs:
           ./hack/test-smoke.sh
 
   multi-node:
-    name: "Multi node (emulated using LXD)"
+    name: "Multi node (emulated using Lima)"
     runs-on: ubuntu-24.04
     timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
         include:
-          - lxc-image: ubuntu:24.04
+          - lima-template: template://ubuntu-24.04
             engine: docker
-# LXD is now banned from pulling images:fedora from https://images.linuxcontainers.org/
-# TODO: switch away from LXD to Incus: https://github.com/rootless-containers/usernetes/pull/332
-#          - lxc-image: images:fedora/39/cloud
-#            engine: podman
+          - lima-template: template://fedora
+            engine: podman
     env:
-      LXC_IMAGE: "${{ matrix.lxc-image }}"
+      LIMA_TEMPLATE: "${{ matrix.lima-template }}"
       CONTAINER_ENGINE: "${{ matrix.engine }}"
     steps:
-      - run: sudo modprobe vxlan
       - uses: actions/checkout@v3
-      - uses: canonical/setup-lxd@v0.1.1
+      - name: "Install QEMU"
+        run: |
+          set -eux
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends ovmf qemu-system-x86 qemu-utils
+          sudo modprobe kvm
+          # `sudo usermod -aG kvm $(whoami)` does not take an effect on GHA
+          sudo chown $(whoami) /dev/kvm
+
+      - name: "Install Lima"
+        run: |
+          set -eux
+          LIMA_VERSION=$(curl -fsSL https://api.github.com/repos/lima-vm/lima/releases/latest | jq -r .tag_name)
+          curl -fsSL https://github.com/lima-vm/lima/releases/download/v${LIMA_VERSION}/lima-${LIMA_VERSION}-Linux-x86_64.tar.gz | sudo tar Cxzvf /usr/local -
+
+      - name: "Cache ~/.cache/lima"
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/lima
+          key: lima-${{ env.LIMA_VERSION }}
+
       - name: "Relax disk pressure limit"
         run: |
           set -x
           sudo snap install yq
           yq -i 'select(.kind=="KubeletConfiguration").evictionHard."imagefs.available"="3Gi"' kubeadm-config.yaml
-      - run: ./hack/create-cluster-lxd.sh
+      - run: ./hack/create-cluster-lima.sh
       - run: kubectl taint nodes --all node-role.kubernetes.io/control-plane- || true
       - run: ./hack/test-smoke.sh
       - if: failure()
@@ -93,16 +110,19 @@ jobs:
           set -x
           kubectl get nodes -o wide
           kubectl get nodes -o yaml
-          ssh -F ${HOME}/.u7s-ci-hosts/ssh_config host0 -- df -h
-          ssh -F ${HOME}/.u7s-ci-hosts/ssh_config host1 -- df -h
+          limactl shell host0 df -h
+          limactl shell host1 df -h
       - if: failure()
         name: "kubectl get pods"
         run: |
           set -x
           kubectl get pods -A -o yaml
-          ssh -F ${HOME}/.u7s-ci-hosts/ssh_config host0 -- journalctl --user --no-pager --since "10 min ago"
+          limactl shell host0 journalctl --user --no-pager --since "10 min ago"
       - name: "Test data persistency after restarting the node"
         run: |
-          lxc restart host0 host1
+          limactl stop host0
+          limactl stop host1
+          limactl start host0
+          limactl start host1
           sleep 30
           ./hack/test-smoke.sh

hack/create-cluster-lima.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+set -eux -o pipefail
+
+: "${LIMACTL:=limactl --tty=false}"
+: "${LIMA_TEMPLATE:=template://default}"
+: "${CONTAINER_ENGINE:=docker}"
+
+guest_home="/home/${USER}.linux"
+
+# Create Rootless Docker hosts
+for host in host0 host1; do
+	# Set --plain to minimize Limaism
+	${LIMACTL} start --plain --network lima:user-v2 --name="${host}" "${LIMA_TEMPLATE}"
+	${LIMACTL} copy -r "$(pwd)" "${host}:${guest_home}/usernetes"
+	${LIMACTL} shell "${host}" sudo CONTAINER_ENGINE="${CONTAINER_ENGINE}" "${guest_home}/usernetes/init-host/init-host.root.sh"
+	# Terminate the current session so that the cgroup delegation takes an effect. This command exits with status 255 as SSH terminates.
+	${LIMACTL} shell "${host}" sudo loginctl terminate-user "${USER}" || true
+	${LIMACTL} shell "${host}" sudo loginctl enable-linger "${USER}"
+	# TODO: disable sudo here
+	${LIMACTL} shell "${host}" CONTAINER_ENGINE="${CONTAINER_ENGINE}" "${guest_home}/usernetes/init-host/init-host.rootless.sh"
+done
+
+# Launch a Kubernetes node inside a Rootless Docker host
+for host in host0 host1; do
+	${LIMACTL} shell "${host}" CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" up
+done
+
+# Bootstrap a cluster with host0
+${LIMACTL} shell host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" kubeadm-init install-flannel kubeconfig join-command
+
+# Let host1 join the cluster
+${LIMACTL} copy host0:~/usernetes/join-command host1:~/usernetes/join-command
+${LIMACTL} shell host1 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" kubeadm-join
+${LIMACTL} shell host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C "${guest_home}/usernetes" sync-external-ip
+
+# Enable kubectl
+ssh -q -f -N -L 6443:127.0.0.1:6443 -F ~/.lima/host0/ssh.config lima-host0
+${LIMACTL} copy host0:${guest_home}/usernetes/kubeconfig ./kubeconfig
+KUBECONFIG="$(pwd)/kubeconfig"
+export KUBECONFIG
+kubectl get nodes -o wide
+kubectl get pods -A

init-host/init-host.root.sh

@@ -22,23 +22,18 @@ cat >/etc/modules-load.d/usernetes.conf <<EOF
 br_netfilter
 vxlan
 EOF
-# systemd-modules-load.service may fail inside LXC
-systemctl restart systemd-modules-load.service || true
+systemctl restart systemd-modules-load.service
 
 cat >/etc/sysctl.d/99-usernetes.conf <<EOF
 # For VXLAN, net.ipv4.conf.default.rp_filter must not be 1 (strict) in the daemon's netns.
 # It may still remain 1 in the host netns, but there is no robust and simple way to
 # configure sysctl for the daemon's netns. So we are configuring it globally here.
 net.ipv4.conf.default.rp_filter = 2
 EOF
-# sysctl may fail inside LXC
-sysctl --system || true
+sysctl --system
 
 if command -v dnf >/dev/null 2>&1; then
 	dnf install -y git shadow-utils make jq
-	# Workaround: SUID bit on newuidmap is dropped on LXC images:fedora/38/cloud,
-	# so it has to be reinstalled
-	dnf reinstall -y shadow-utils
 else
 	apt-get update
 	apt-get install -y git uidmap make jq