Merge pull request #67 from jcvenegas/static-qemu

pkg: Build an static qemu for kata

.ci/test.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+CI=${CI:-}
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+readonly toplevel_mk="${script_dir}/../Makefile"
+source "${script_dir}/lib.sh"
+
+make_target() {
+	target=$1
+	dir=$2
+
+	pushd "${script_dir}/.." >> /dev/null
+	if [ -n "${CI}" ] &&  ! git whatchanged  origin/master..HEAD  "${dir}" | grep "${dir}" >> /dev/null; then
+		echo "Not changes in ${dir}"
+		return
+	fi
+	popd >> /dev/null
+	echo "Changes found in $dir"
+	make -f "${toplevel_mk}" ${target}
+}
+
+make_target test-release-tools "release/"
+make_target test-packaging-tools "obs-packaging/"
+make_target test-static-build "static-build/"

Makefile

@@ -8,15 +8,14 @@
 MK_DIR :=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 .PHONY: test test-release-tools
 
-test: test-release-tools test-packaging-tools
+test:
+	@$(MK_DIR)/.ci/test.sh
 
 test-release-tools:
 	@$(MK_DIR)/release/tag_repos_test.sh
 
+test-static-build:
+	@make -f $(MK_DIR)/static-build/qemu/Makefile
+
 test-packaging-tools:
-ifndef CI
 	@$(MK_DIR)/build_from_docker.sh
-else
-	@echo "Skip test-packaging-tools"
-	@echo "See: https://github.com/kata-containers/packaging/issues/68"
-endif

scripts/configure-hypervisor.sh

@@ -55,6 +55,7 @@ Options:
 	     is specified.
 	-h : Display this help.
 	-m : Display options one per line (includes continuation characters).
+	-s : Generate options to build static
 
 Example:
 
@@ -185,24 +186,6 @@ show_array()
 # Entry point
 main()
 {
-	local qemu_version_file="VERSION"
-	[ -f ${qemu_version_file} ] || die "QEMU version file '$qemu_version_file' not found"
-
-	local qemu_version_major=$(cut -d. -f1 "${qemu_version_file}")
-	local qemu_version_minor=$(cut -d. -f2 "${qemu_version_file}")
-
-	[ -n "${qemu_version_major}" ] \
-		|| die "cannot determine qemu major version from file $qemu_version_file"
-	[ -n "${qemu_version_minor}" ] \
-		|| die "cannot determine qemu minor version from file $qemu_version_file"
-
-	local gcc_version_major=$(gcc -dumpversion | cut -f1 -d.)
-	local gcc_version_minor=$(gcc -dumpversion | cut -f2 -d.)
-
-	[ -n "${gcc_version_major}" ] \
-		|| die "cannot determine gcc major version, please ensure it is installed"
-	[ -n "${gcc_version_minor}" ] \
-		|| die "cannot determine gcc minor version, please ensure it is installed"
 
 	arch=$(arch)
 
@@ -222,7 +205,7 @@ main()
 
 	action=""
 
-	while getopts "dhm" opt
+	while getopts "dhms" opt
 	do
 		case "$opt" in
 			d)
@@ -237,6 +220,9 @@ main()
 			m)
 				action="multi"
 				;;
+			s)
+				static="true"
+				;;
 		esac
 	done
 
@@ -245,6 +231,25 @@ main()
 	[ -z "$1" ] && die "need hypervisor name"
 	hypervisor="$1"
 
+	local qemu_version_file="VERSION"
+	[ -f ${qemu_version_file} ] || die "QEMU version file '$qemu_version_file' not found"
+
+	local qemu_version_major=$(cut -d. -f1 "${qemu_version_file}")
+	local qemu_version_minor=$(cut -d. -f2 "${qemu_version_file}")
+
+	[ -n "${qemu_version_major}" ] \
+		|| die "cannot determine qemu major version from file $qemu_version_file"
+	[ -n "${qemu_version_minor}" ] \
+		|| die "cannot determine qemu minor version from file $qemu_version_file"
+
+	local gcc_version_major=$(gcc -dumpversion | cut -f1 -d.)
+	local gcc_version_minor=$(gcc -dumpversion | cut -f2 -d.)
+
+	[ -n "${gcc_version_major}" ] \
+		|| die "cannot determine gcc major version, please ensure it is installed"
+	[ -n "${gcc_version_minor}" ] \
+		|| die "cannot determine gcc minor version, please ensure it is installed"
+
 	#---------------------------------------------------------------------
 	# Disabled options
 
@@ -303,6 +308,10 @@ main()
 		qemu_options+=(security:--disable-static)
 	fi
 
+	if [ -n ${static} ]; then
+		qemu_options+=(misc:--static)
+	fi
+
 	# Not required as "-uuid ..." is always passed to the qemu binary
 	qemu_options+=(size:--disable-uuid)
 
@@ -347,7 +356,7 @@ main()
 	fi
 
 	# Support Ceph RADOS Block Device (RBD)
-	qemu_options+=(functionality:--enable-rbd)
+	[ -z "${static}" ] && qemu_options+=(functionality:--enable-rbd)
 
 	# In "passthrough" security mode
 	# (-fsdev "...,security_model=passthrough,..."), qemu uses a helper
@@ -397,7 +406,7 @@ main()
 	# SECURITY: Link binary as a Position Independant Executable,
 	# and take advantage of ASLR, making ROP attacks much harder to perform.
 	# (https://wiki.debian.org/Hardening)
-	_qemu_ldflags+=" -pie"
+	[ -z "${static}" ] && _qemu_ldflags+=" -pie"
 
 	# SECURITY: Disallow executing code on the stack.
 	_qemu_ldflags+=" -z noexecstack"

scripts/lib.sh

@@ -0,0 +1,55 @@
+readonly kata_arch_sh="${GOPATH}/src/github.com/kata-containers/tests/.ci/kata-arch.sh"
+
+get_kata_arch(){
+	go get -u github.com/kata-containers/tests || true
+	[ -f "${kata_arch_sh}" ] || die "Not found $kata_arch_sh"
+}
+
+install_yq() {
+	GOPATH=${GOPATH:-${HOME}/go}
+	local yq_path="${GOPATH}/bin/yq"
+	local yq_pkg="github.com/mikefarah/yq"
+	[ -x  "${GOPATH}/bin/yq" ] && return
+
+	get_kata_arch
+	goarch=$("${kata_arch_sh}" -g)
+
+	mkdir -p "${GOPATH}/bin"
+
+	# Workaround to get latest release from github (to not use github token).
+	# Get the redirection to latest release on github.
+	yq_latest_url=$(curl -Ls -o /dev/null -w %{url_effective} "https://${yq_pkg}/releases/latest")
+	# The redirected url should include the latest release version
+	# https://github.com/mikefarah/yq/releases/tag/<VERSION-HERE>
+	yq_version=$(basename "${yq_latest_url}")
+
+
+	local yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_linux_${goarch}"
+	curl -o "${yq_path}" -L "${yq_url}"
+	chmod +x "${yq_path}"
+}
+
+get_from_kata_deps(){
+	dependency="$1"
+	GOPATH=${GOPATH:-${HOME}/go}
+	# This is needed in order to retrieve the version for qemu-lite
+	install_yq >&2
+	runtime_repo="github.com/kata-containers/runtime"
+	runtime_repo_dir="$GOPATH/src/${runtime_repo}"
+	versions_file="${runtime_repo_dir}/versions.yaml"
+	mkdir -p $(dirname "${runtime_repo_dir}")
+	[ -d "${runtime_repo_dir}" ] ||  git clone --quiet https://${runtime_repo}.git "${runtime_repo_dir}"
+	[ ! -f "$versions_file" ] && { echo >&2 "ERROR: cannot find $versions_file"; exit 1; }
+	result=$("${GOPATH}/bin/yq" read "$versions_file" "$dependency")
+	[ "$result" = "null" ] && result=""
+	echo "$result"
+}
+
+die() {
+		echo >&2 "ERROR: $*"
+			exit 1
+}
+
+info() {
+		echo >&2 "INFO: $*"
+}

static-build/qemu/Dockerfile

@@ -0,0 +1,46 @@
+from ubuntu:16.04
+
+ARG QEMU_REPO
+# commit/tag/branch
+ARG QEMU_VERSION
+
+WORKDIR /root/qemu
+RUN apt-get update
+RUN apt-get install -y \
+	    autoconf \
+	    automake \
+	    bc \
+	    bison \
+	    cpio \
+	    flex \
+	    gawk \
+	    libaudit-dev \
+	    libcap-dev \
+	    libcap-ng-dev \
+	    libdw-dev \
+	    libelf-dev \
+	    libglib2.0-0 \
+	    libglib2.0-dev \
+	    libglib2.0-dev git \
+	    libltdl-dev \
+	    libpixman-1-dev \
+	    libtool \
+	    pkg-config \
+	    pkg-config \
+	    python \
+	    python-dev \
+	    rsync \
+	    zlib1g-dev
+
+RUN cd  .. && git clone "${QEMU_REPO}" qemu
+RUN git checkout "${QEMU_VERSION}"
+RUN git clone https://github.com/qemu/capstone.git capstone
+RUN git clone  https://github.com/qemu/keycodemapdb.git ui/keycodemapdb
+
+ADD configure-hypervisor.sh /root/configure-hypervisor.sh
+
+RUN /root/configure-hypervisor.sh -s kata-qemu | xargs ./configure --prefix=/opt/kata --with-pkgversion=kata-static
+RUN make clean
+RUN make -j$(nproc)
+RUN make install DESTDIR=/tmp/qemu-static
+RUN cd /tmp/qemu-static && tar -czvf kata-qemu-static.tar.gz *

static-build/qemu/Makefile

@@ -0,0 +1,8 @@
+MK_DIR :=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+CONFIG_DIR := $(MK_DIR)/../../scripts/
+
+build:
+	"$(MK_DIR)/build-static-qemu.sh"
+
+clean:
+	rm -f kata-qemu-static.tar.gz

static-build/qemu/build-static-qemu.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+source "${script_dir}/../../scripts/lib.sh"
+
+config_dir="${script_dir}/../../scripts/"
+
+qemu_repo="${qemu_repo:-}"
+qemu_version="${qemu_version:-}"
+
+if [ -z "$qemu_repo" ]; then
+	info "Get qemu information from runtime versions.yaml"
+	qemu_url=$(get_from_kata_deps "assets.hypervisor.qemu.url")
+	[ -n "$qemu_url" ] || die "failed to get qemu url"
+	qemu_repo="${qemu_url}.git"
+fi
+[ -n "$qemu_repo" ] ||  die "failed to get qemu repo"
+
+
+[ -n "$qemu_version" ] || qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
+[ -n "$qemu_version" ] ||  die "failed to get qemu version"
+
+info "Build ${qemu_repo} version: ${qemu_version}"
+
+http_proxy="${http_proxy:-}"
+https_proxy="${https_proxy:-}"
+
+docker build \
+	--build-arg http_proxy="${http_proxy}" \
+	--build-arg https_proxy="${https_proxy}" \
+	--build-arg QEMU_REPO="${qemu_repo}" \
+	--build-arg QEMU_VERSION="${qemu_version}" \
+	"${config_dir}" \
+	-f "${script_dir}/Dockerfile" \
+	-t qemu-static
+
+docker run \
+	-i \
+	-v "${PWD}":/share qemu-static \
+	mv /tmp/qemu-static/kata-qemu-static.tar.gz /share/