Support CONTAINER_ENGINE=(podman|nerdctl) in addition to docker

- Dockerfile: Use long image names with the `docker.io/` prefix for
  compatibility with Red Hat's containers
- Dockerfile: Avoid using BuildKit features that are unsupported by buildah
- docker-compose.yaml: Remove `tty: true`, as it is unsupported by `nerdctl compose up -d`.
- Makefile: Remove `$(COMPOSE) cp`, as it is unsupported by podman-compose

nerdctl support depends on:
- containerd/nerdctl PR 2507 (milestone: nerdctl v1.5.1)

Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>

.github/workflows/main.yaml

@@ -9,6 +9,12 @@ jobs:
     name: "Single node"
     runs-on: ubuntu-22.04
     timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      matrix:
+        engine: [docker, podman]
+    env:
+      CONTAINER_ENGINE: "${{ matrix.engine }}"
     steps:
       - uses: actions/checkout@v3
       - name: Set up cgroup v2 delegation
@@ -19,15 +25,26 @@ jobs:
           Delegate=cpu cpuset io memory pids
           EOF
           sudo systemctl daemon-reload
+      - name: Remove preinstalled Moby
+        # Preinstalled Moby does not contain dockerd-rootless-setuptool.sh
+        run: sudo apt-get remove moby-engine-*
       - name: Set up Rootless Docker
+        if: ${{ matrix.engine == 'docker' }}
         run: |
           set -eux -o pipefail
-          sudo apt-get remove moby-engine-*
           curl https://get.docker.com | sudo sh
           sudo systemctl disable --now docker.socket docker.service
           sudo rm -rf /var/run/docker*
           dockerd-rootless-setuptool.sh install
           docker info
+      - name: Set up Rootless Podman
+        if: ${{ matrix.engine == 'podman' }}
+        run: |
+          set -eux -o pipefail
+          # Preinstalled Podman is too old (v3.4.4)
+          sudo apt-get remove podman*
+          sudo ./init-host/init-host.root.d/install-podman.sh
+          podman info
       - run: make up
       - run: sleep 5
       - run: make kubeadm-init
@@ -46,6 +63,17 @@ jobs:
     name: "Multi node (emulated using LXD)"
     runs-on: ubuntu-22.04
     timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - lxc-image: ubuntu:22.04
+            engine: docker
+          - lxc-image: images:fedora/38/cloud
+            engine: podman
+    env:
+      LXC_IMAGE: "${{ matrix.lxc-image }}"
+      CONTAINER_ENGINE: "${{ matrix.engine }}"
     steps:
       - run: sudo modprobe vxlan
       - uses: actions/checkout@v3

Dockerfile

@@ -1,19 +1,11 @@
-ARG BASE_IMAGE=kindest/node:v1.28.0
-
-# TODO: use `ADD --checksum=sha256...`
-FROM scratch AS cni-plugins-amd64
-ADD https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz /cni-plugins.tgz
-
-FROM scratch AS cni-plugins-arm64
-ADD https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-arm64-v1.3.0.tgz /cni-plugins.tgz
-
-ARG TARGETARCH
-FROM cni-plugins-$TARGETARCH AS cni-plugins
-
-ARG BASE_IMAGE
+ARG BASE_IMAGE=docker.io/kindest/node:v1.28.0
+ARG CNI_PLUGINS_VERSION=v1.3.0
 FROM ${BASE_IMAGE}
-RUN --mount=type=bind,from=cni-plugins,dst=/mnt/tmp \
-  tar Cxzvf /opt/cni/bin /mnt/tmp/cni-plugins.tgz
+# TODO: check SHA256SUMS of cni-plugins
+ARG CNI_PLUGINS_VERSION
+RUN arch="$(uname -m | sed -e s/x86_64/amd64/ -e s/aarch64/arm64/)" && \
+  curl -fsSL https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${arch}-${CNI_PLUGINS_VERSION}.tgz \
+  | tar Cxzv /opt/cni/bin
 # gettext-base: for `envsubst`
 # moreutils: for `sponge`
 # socat: for `socat` (to silence "[WARNING FileExisting-socat]" from kubeadm)

Makefile

@@ -17,11 +17,14 @@ export U7S_NODE_NAME:= $(NODE_NAME)
 # Not accessible from other hosts.
 export U7S_NODE_SUBNET := $(NODE_SUBNET)
 
-DOCKER ?= docker
+CONTAINER_ENGINE ?= $(shell $(CURDIR)/Makefile.d/detect_container_engine.sh CONTAINER_ENGINE)
+export CONTAINER_ENGINE := $(CONTAINER_ENGINE)
 
-export DOCKER := $(DOCKER)
+CONTAINER_ENGINE_TYPE ?= $(shell $(CURDIR)/Makefile.d/detect_container_engine.sh CONTAINER_ENGINE_TYPE)
+export CONTAINER_ENGINE_TYPE := $(CONTAINER_ENGINE_TYPE)
+
+COMPOSE ?= $(shell $(CURDIR)/Makefile.d/detect_container_engine.sh COMPOSE)
 
-COMPOSE := $(DOCKER) compose
 NODE_SERVICE_NAME := node
 NODE_SHELL := $(COMPOSE) exec \
 	-e U7S_HOST_IP=$(U7S_HOST_IP) \
@@ -78,7 +81,7 @@ logs:
 
 .PHONY: kubeconfig
 kubeconfig:
-	$(COMPOSE) cp $(NODE_SERVICE_NAME):/etc/kubernetes/admin.conf ./kubeconfig
+	$(COMPOSE) exec -T $(NODE_SERVICE_NAME) cat /etc/kubernetes/admin.conf >kubeconfig
 	@echo "# Run the following command by yourself:"
 	@echo "export KUBECONFIG=$(shell pwd)/kubeconfig"
 ifeq ($(shell command -v kubectl 2> /dev/null),)
@@ -88,7 +91,7 @@ endif
 
 .PHONY: kubectl
 kubectl:
-	$(COMPOSE) cp $(NODE_SERVICE_NAME):/usr/bin/kubectl ./kubectl
+	$(COMPOSE) exec -T --workdir=/usr/bin $(NODE_SERVICE_NAME) tar c kubectl | tar xv
 	@echo "# Run the following command by yourself:"
 	@echo "export PATH=$(shell pwd):\$$PATH"
 	@echo "source <(kubectl completion bash)"

Makefile.d/check-preflight.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 set -eu
 
+function INFO() {
+	echo >&2 -e "\e[104m\e[97m[INFO]\e[49m\e[39m $@"
+}
 function WARNING() {
 	echo >&2 -e "\e[101m\e[97m[WARNING]\e[49m\e[39m $@"
 }
@@ -9,12 +12,21 @@ function ERROR() {
 	echo >&2 -e "\e[101m\e[97m[ERROR]\e[49m\e[39m $@"
 }
 
-: "${DOCKER:=docker}"
+script_dir="$(dirname "$0")"
+detect_engine="${script_dir}"/detect_container_engine.sh
+: "${CONTAINER_ENGINE:=$("${detect_engine}" CONTAINER_ENGINE)}"
+: "${CONTAINER_ENGINE_TYPE:=$("${detect_engine}" CONTAINER_ENGINE_TYPE)}"
 : "${QUICK:=0}"
-: "${BUSYBOX_IMAGE:=busybox}"
+: "${BUSYBOX_IMAGE:=docker.io/library/busybox:latest}"
+
+if [ -z "${CONTAINER_ENGINE}" ] || [ -z "${CONTAINER_ENGINE_TYPE}" ]; then
+	ERROR "No container engine was detected"
+	exit 1
+fi
+INFO "Detected container engine type: ${CONTAINER_ENGINE_TYPE}"
 
 # Check hard dependency commands
-for f in make jq "${DOCKER}"; do
+for f in make jq "${CONTAINER_ENGINE}"; do
 	if ! command -v "${f}" >/dev/null 2>&1; then
 		ERROR "Command \"${f}\" is not installed"
 		exit 1
@@ -28,9 +40,22 @@ for f in kubectl; do
 	fi
 done
 
-# Check if Docker is running in Rootless mode
-# TODO: support Podman?
-if "${DOCKER}" info --format '{{json .SecurityOptions}}' | grep -q "name=rootless"; then
+rootless=
+case "${CONTAINER_ENGINE_TYPE}" in
+"podman")
+	if [ "$(${CONTAINER_ENGINE} info --format '{{.Host.Security.Rootless}}')" = "true" ]; then
+		rootless=1
+	fi
+	;;
+*)
+	if ${CONTAINER_ENGINE} info --format '{{json .SecurityOptions}}' | grep -q "name=rootless"; then
+		rootless=1
+	fi
+	;;
+esac
+
+# Check if the container engine is running in Rootless mode
+if [ "${rootless}" = "1" ]; then
 	# Check systemd lingering: https://rootlesscontaine.rs/getting-started/common/login/
 	if command -v loginctl >/dev/null 2>&1; then
 		if [ "$(loginctl list-users --output json | jq ".[] | select(.uid == "${UID}").linger")" != "true" ]; then
@@ -57,7 +82,7 @@ if "${DOCKER}" info --format '{{json .SecurityOptions}}' | grep -q "name=rootles
 		fi
 	fi
 else
-	WARNING "Docker does not seem running in Rootless mode"
+	WARNING "Container engine (${CONTAINER_ENGINE}) does not seem running in Rootless mode"
 fi
 
 # Check kernel modules
@@ -68,10 +93,10 @@ for f in br_netfilter ip6_tables ip6table_nat ip_tables iptable_nat vxlan; do
 done
 
 if [ "$QUICK" != "1" ]; then
-	# Check net.ipv4.conf.default.rp_filter in the daemon's network namespace.
+	# Check net.ipv4.conf.default.rp_filter in the container engine's network namespace. (e.g., netns of dockerd)
 	# The value can be 0 (disabled) or 2 (loose), must not be 1 (strict).
-	if [ "$(${DOCKER} run --rm --net=host "${BUSYBOX_IMAGE}" sysctl -n net.ipv4.conf.default.rp_filter)" == "1" ]; then
-		ERROR "sysctl value \"net.ipv4.conf.default.rp_filter\" must be 0 (disabled) or 2 (loose) in the daemon's network namespace"
+	if [ "$(${CONTAINER_ENGINE} run --rm --net=host "${BUSYBOX_IMAGE}" sysctl -n net.ipv4.conf.default.rp_filter)" == "1" ]; then
+		ERROR "sysctl value \"net.ipv4.conf.default.rp_filter\" must be 0 (disabled) or 2 (loose) in the container engine's network namespace"
 		exit 1
 	fi
 fi

Makefile.d/detect_container_engine.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+set -eu -o pipefail
+: "${CONTAINER_ENGINE:=}"
+: "${COMPOSE:=}"
+
+if [ -z "${CONTAINER_ENGINE}" ]; then
+	if command -v dockerd-rootless.sh >/dev/null 2>&1; then
+		CONTAINER_ENGINE=docker
+	elif command -v containerd-rootless.sh >/dev/null 2>&1; then
+		CONTAINER_ENGINE=nerdctl
+	elif command -v podman >/dev/null 2>&1; then
+		CONTAINER_ENGINE=podman
+	else
+		echo >&2 "$0: no container engine was detected"
+		exit 1
+	fi
+fi
+
+CONTAINER_ENGINE_TYPE=docker
+if [[ "${CONTAINER_ENGINE}" = *"podman"* ]]; then
+	CONTAINER_ENGINE_TYPE=podman
+elif [[ "${CONTAINER_ENGINE}" = *"nerdctl"* ]]; then
+	CONTAINER_ENGINE_TYPE=nerdctl
+fi
+
+if [ -z "${COMPOSE}" ]; then
+	COMPOSE="${CONTAINER_ENGINE} compose"
+	if [ "${CONTAINER_ENGINE_TYPE}" = "podman" ]; then
+		COMPOSE=podman-compose
+	fi
+fi
+
+case "$#" in
+0)
+	echo "CONTAINER_ENGINE=${CONTAINER_ENGINE}"
+	echo "CONTAINER_ENGINE_TYPE=${CONTAINER_ENGINE_TYPE}"
+	echo "COMPOSE=${COMPOSE}"
+	;;
+1)
+	case "$1" in
+	"CONTAINER_ENGINE" | "CONTAINER_ENGINE_TYPE" | "COMPOSE")
+		echo "${!1}"
+		;;
+	*)
+		echo >&2 "$0: unknown argument: $1"
+		exit 1
+		;;
+	esac
+	;;
+*)
+	echo >&2 "$0: too many arguments"
+	exit 1
+	;;
+esac

README.md

@@ -22,12 +22,23 @@ but Usernetes (Gen 2) supports creating a cluster with multiple hosts.
 
 ## Requirements
 
-- Host OS should be one of the following:
-  - Ubuntu 22.04 (recommended)
-  - Rocky Linux 9
-  - AlmaLinux 9
+- One of the following host operating system:
+
+|Host operating system|Minimum version|
+|---------------------|---------------|
+|Ubuntu (recommended) |22.04          |
+|Rocky Linux          |9              |
+|AlmaLinux            |9              |
+|Fedora               |(?)            |
+
+- One of the following container engines:
+
+|Container Engine                                                                    |Minimum version|
+|------------------------------------------------------------------------------------|---------------|
+|[Rootless Docker](https://rootlesscontaine.rs/getting-started/docker/) (recommended)|v20.10         |
+|[Rootless Podman](https://rootlesscontaine.rs/getting-started/podman/)              |v4.x           |
+|[Rootless nerdctl](https://rootlesscontaine.rs/getting-started/containerd/)         |v1.5.1         |
 
-- [Rootless Docker](https://rootlesscontaine.rs/getting-started/docker/):
 ```bash
 curl -o install.sh -fsSL https://get.docker.com
 sudo sh install.sh
@@ -98,6 +109,9 @@ make down-v
 kubectl taint nodes --all node-role.kubernetes.io/control-plane-
 ```
 
+The container engine defaults to Docker.
+To change the container engine, set `export CONTAINER_ENGINE=podman` or `export CONTAINER_ENGINE=nerdctl`.
+
 ## Limitations
 - Node ports cannot be exposed automatically. Edit [`docker-compose.yaml`](./docker-compose.yaml) for exposing additional node ports.
 - Most of host files are not visible with `hostPath` mounts. Edit [`docker-compose.yaml`](./docker-compose.yaml) for mounting additional files.

docker-compose.yaml

@@ -7,7 +7,6 @@ services:
     hostname: ${U7S_NODE_NAME}
     privileged: true
     restart: on-failure
-    tty: true
     ports:
       # etcd
       - 2379:2379

hack/create-cluster-lxd.sh

@@ -1,28 +1,30 @@
 #!/bin/bash
 set -eux -o pipefail
 
+: "${CONTAINER_ENGINE:=docker}"
+
 # Create Rootless Docker hosts
 ./hack/create-hosts-lxd.sh "${HOME}/.u7s-ci-hosts" host0 host1
 SCP="scp -F ${HOME}/.u7s-ci-hosts/ssh_config"
 SSH="ssh -F ${HOME}/.u7s-ci-hosts/ssh_config"
 for host in host0 host1; do
 	$SCP -r "$(pwd)" "${host}:~/usernetes"
-	$SSH "${USER}-sudo@${host}" sudo "~${USER}/usernetes/init-host/init-host.root.sh"
+	$SSH "${USER}-sudo@${host}" sudo CONTAINER_ENGINE="${CONTAINER_ENGINE}" "~${USER}/usernetes/init-host/init-host.root.sh"
 	$SSH "${USER}-sudo@${host}" sudo loginctl enable-linger "${USER}"
-	$SSH "${host}" ~/usernetes/init-host/init-host.rootless.sh
+	$SSH "${host}" CONTAINER_ENGINE="${CONTAINER_ENGINE}" ~/usernetes/init-host/init-host.rootless.sh
 done
 
 # Launch a Kubernetes node inside a Rootless Docker host
 for host in host0 host1; do
-	$SSH "${host}" make -C ~/usernetes up
+	$SSH "${host}" CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes up
 done
 
 # Bootstrap a cluster with host0
-$SSH host0 make -C ~/usernetes kubeadm-init install-flannel kubeconfig join-command
+$SSH host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes kubeadm-init install-flannel kubeconfig join-command
 
 # Let host1 join the cluster
 $SCP host0:~/usernetes/join-command host1:~/usernetes/join-command
-$SSH host1 make -C ~/usernetes kubeadm-join
+$SSH host1 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes kubeadm-join
 
 # Enable kubectl
 $SCP host0:~/usernetes/kubeconfig ./kubeconfig

hack/create-hosts-lxd.sh

@@ -8,6 +8,7 @@ dir=$1
 shift
 names=$*
 
+: "${LXC_IMAGE:="ubuntu:22.04"}"
 LXC="sudo lxc"
 
 echo "USER=${USER}"
@@ -42,7 +43,7 @@ EOF
 fi
 
 for name in ${names}; do
-	${LXC} init ubuntu:22.04 "${name}" -c security.privileged=true -c security.nesting=true
+	${LXC} init "${LXC_IMAGE}" "${name}" -c security.privileged=true -c security.nesting=true
 	${LXC} config device add "${name}" bind-boot disk source=/boot path=/boot readonly=true
 	${LXC} config set "${name}" user.user-data - <"${userdata}"
 	${LXC} start "${name}"

init-host/init-host.root.d/install-podman.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# This script installs the latest release of Podman.
+# Repository information is from https://podman.io/docs/installation#linux-distributions
+set -eux -o pipefail
+if [ "$(id -u)" != "0" ]; then
+	echo "Must run as the root"
+	exit 1
+fi
+
+if command -v dnf >/dev/null 2>&1; then
+	sudo dnf install -y podman podman-compose
+else
+	mkdir -p /etc/apt/keyrings
+	curl -fsSL "https://download.opensuse.org/repositories/devel:kubic:libcontainers:unstable/xUbuntu_$(lsb_release -rs)/Release.key" |
+		gpg --dearmor |
+		tee /etc/apt/keyrings/devel_kubic_libcontainers_unstable.gpg >/dev/null
+	echo \
+		"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/devel_kubic_libcontainers_unstable.gpg]\
+        https://download.opensuse.org/repositories/devel:kubic:libcontainers:unstable/xUbuntu_$(lsb_release -rs)/ /" |
+		tee /etc/apt/sources.list.d/devel:kubic:libcontainers:unstable.list >/dev/null
+	apt-get update -qq
+	apt-get -qq -y install podman
+	# No dpkg for podman-compose ?
+	pip3 install podman-compose
+fi