From a27fb7102ad69873f77d67eb13bc7b6ec3ade81a Mon Sep 17 00:00:00 2001 From: Cristian Kubis Date: Wed, 17 Nov 2021 15:14:55 +0100 Subject: [PATCH] Add node upgrade logic for the controller Signed-off-by: Cristian Kubis --- controller/Dockerfile | 1 + controller/container_node_control_tool.py | 12 +++- controller/restart_indy_node.sh | 8 +-- controller/upgrade_indy.sh | 87 +++++++++++++++++++++++ test/docker-compose.yaml | 37 ++++++---- test/init-test-network.sh | 11 ++- 6 files changed, 136 insertions(+), 20 deletions(-) create mode 100755 controller/upgrade_indy.sh diff --git a/controller/Dockerfile b/controller/Dockerfile index 5ba6e35..758dfe5 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -28,6 +28,7 @@ RUN apt-get update -y && apt-get install -y podman RUN mkdir /opt/controller COPY restart_indy_node.sh /opt/controller/restart_indy_node +COPY upgrade_indy.sh /opt/controller/upgrade_indy COPY start_node_control_tool /opt/controller/start_node_control_tool COPY container_node_control_tool.py /opt/controller/container_node_control_tool.py COPY init_and_run.sh ./ diff --git a/controller/container_node_control_tool.py b/controller/container_node_control_tool.py index fc582da..a93e2cb 100644 --- a/controller/container_node_control_tool.py +++ b/controller/container_node_control_tool.py @@ -81,7 +81,7 @@ def _listen(self): self.server.setblocking(0) # Bind the socket to the port - self.server_address = ('localhost', 30003) + self.server_address = ('0.0.0.0', 30003) logger.info('Node control tool is starting up on {} port {}'.format(*self.server_address)) self.server.bind(self.server_address) @@ -94,6 +94,11 @@ def _call_restart_node_script(self): cmd = compose_cmd(['/opt/controller/restart_indy_node']) NodeControlUtil.run_shell_script(cmd, timeout=self.timeout) + def _call_upgrade_node_script(self): + logger.info('Upgrading indy') + cmd = compose_cmd(['/opt/controller/upgrade_indy']) + NodeControlUtil.run_shell_script(cmd, timeout=self.timeout) + def _upgrade( self, new_src_ver: SourceVersion, @@ -101,7 +106,10 @@ def _upgrade( migrate=True, rollback=True ): - logger.error("Upgrade message has not yet been implemented") + try: + self._call_upgrade_node_script() + except Exception as ex: + logger.error("Upgrade fail: " + ex.args[0]) def _restart(self): try: diff --git a/controller/restart_indy_node.sh b/controller/restart_indy_node.sh index 3e8f85a..01894a8 100755 --- a/controller/restart_indy_node.sh +++ b/controller/restart_indy_node.sh @@ -4,15 +4,15 @@ print_error() { >&2 echo -e "\033[1;31mError:\033[0m $1" } -if [ -z "$CONTAINER" ]; then - print_error "Missing node container environment variable" +if [ -z "$NODE_CONTAINER" ]; then + print_error "Missing node container name environment variable" exit 22 fi if [ "$ENGINE" == "podman" ]; then - podman --remote container restart $CONTAINER + podman --remote container restart "$NODE_CONTAINER" else - docker container restart $CONTAINER + docker container restart "$NODE_CONTAINER" fi exit 0 diff --git a/controller/upgrade_indy.sh b/controller/upgrade_indy.sh new file mode 100755 index 0000000..f93400b --- /dev/null +++ b/controller/upgrade_indy.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +print_error() { + >&2 echo -e "\033[1;31mError:\033[0m $1" +} + +if [[ -z "$NODE_CONTAINER" ]]; then + print_error "Missing node container name environment variable" + exit 22 +fi + +if [[ -z "$CONTROLLER_CONTAINER" ]]; then + print_error "Missing controller container name environment variable" + exit 22 +fi + +# use docker if no other engine has been specified +ENGINE="${ENGINE:-docker}" + +# make sure the selected container engine is supported +if [[ "$ENGINE" != "podman" && "$ENGINE" != "docker" ]]; then + print_error "Unsupported container engine: $ENGINE" + exit 22 +fi + +for container in $NODE_CONTAINER $CONTROLLER_CONTAINER; do + inspect="$ENGINE container inspect $container --format " + + if [[ "$($inspect '{{.State.Status}}')" != "running" ]]; then + echo "Skipping upgrade because the container '$container' is not running" + continue + fi + + # get name of the image used by the container + image_name="$($inspect '{{.Config.Image}}')" + if [[ -z "$image_name" ]]; then + print_error "Container $container does not exist" + exit 22 + fi + + # save the creation date of the image + image_date="$($ENGINE image inspect "$image_name" --format '{{.Created}}')" + + # try to pull a new image + output="$($ENGINE image pull "$image_name")" + + if [[ $? != 0 ]]; then + print_error "$ENGINE pull failed: $output" + exit 1 + fi + + # compare the old and new creation date to verify that a newer image has been pulled + if [[ "$($ENGINE image inspect $image_name --format '{{.Created}}')" != "$image_date" && "$container" == "$NODE_CONTAINER" ]]; then + # parse and save various container parameter to supply to the new one + binds="-v $($inspect '{{join .HostConfig.Binds " -v "}}')" + ports="$($inspect '{{range $k, $v := .NetworkSettings.Ports}}{{range $v}}{{print "-p " .HostIp ":" .HostPort ":" $k " "}}{{end}}{{end}}' | sed 's/:::/[::]:/g')" + network="$($inspect '{{range $k, $v := .NetworkSettings.Networks}}{{$id := slice "'$($inspect {{.Id}})'" 0 12}}{{$aliases := join $v.Aliases " --network-alias "}}{{println "--network" $k "--ip" $v.IPAddress "--network-alias" $aliases "--network-alias" $id}}{{end}}' | head -n1)" + restart="--restart $($inspect '{{.HostConfig.RestartPolicy.Name}}')" + init="$($inspect '{{.HostConfig.Init}}' | sed 's/true/--init/g; /--init/!s/.*//')" + $inspect '{{join .Config.Env "\n"}}' > /tmp/envs + envs="$(if [ -s /tmp/envs ]; then echo "--env-file /tmp/envs"; else echo ""; fi)" + $inspect '{{range $k,$v := .Config.Labels}}{{printf "%s=%s\n" $k $v}}{{end}}' | grep -v "^org.opencontainers.image" > /tmp/labels + # patch image hash in the docker-compose labels + sed -i "s/sha256.*/$($ENGINE image inspect docker --format '{{.Id}}')/g" /tmp/labels + labels="$(if [ -s /tmp/labels ]; then echo "--label-file /tmp/labels"; else echo ""; fi)" + + output=$($ENGINE container create --name ${container}_new $binds $ports $network $init $restart $envs $labels $image_name) 1> /dev/null + if [[ $? != 0 ]]; then + print_error "failed to create new node container: $output" + exit 1 + fi + + # exit bash script if any of the following fails + set -e + + $ENGINE container stop $container 1> /dev/null + $ENGINE container rm $container 1> /dev/null + $ENGINE container rename ${container}_new $container 1> /dev/null + $ENGINE container start $container 1> /dev/null + + rm /tmp/{envs,labels} + else + echo "$container image is up to date" + fi +done + +exit 0 diff --git a/test/docker-compose.yaml b/test/docker-compose.yaml index bfd9d15..38cbb86 100644 --- a/test/docker-compose.yaml +++ b/test/docker-compose.yaml @@ -9,7 +9,7 @@ services: - "${INDY_NODE_PORT_1}:${INDY_NODE_PORT_1}" - "${INDY_CLIENT_PORT_1}:${INDY_CLIENT_PORT_1}" volumes: - - ./etc_indy:/etc/indy + - ./etc_indy/node1:/etc/indy - ./lib_indy:/var/lib/indy environment: - INDY_NODE_NAME=Node1 @@ -32,7 +32,7 @@ services: - "${INDY_NODE_PORT_2}:${INDY_NODE_PORT_2}" - "${INDY_CLIENT_PORT_2}:${INDY_CLIENT_PORT_2}" volumes: - - ./etc_indy:/etc/indy + - ./etc_indy/node2:/etc/indy - ./lib_indy:/var/lib/indy environment: - INDY_NODE_NAME=Node2 @@ -55,7 +55,7 @@ services: - "${INDY_NODE_PORT_3}:${INDY_NODE_PORT_3}" - "${INDY_CLIENT_PORT_3}:${INDY_CLIENT_PORT_3}" volumes: - - ./etc_indy:/etc/indy + - ./etc_indy/node3:/etc/indy - ./lib_indy:/var/lib/indy environment: - INDY_NODE_NAME=Node3 @@ -78,7 +78,7 @@ services: - "${INDY_NODE_PORT_4}:${INDY_NODE_PORT_4}" - "${INDY_CLIENT_PORT_4}:${INDY_CLIENT_PORT_4}" volumes: - - ./etc_indy:/etc/indy + - ./etc_indy/node4:/etc/indy - ./lib_indy:/var/lib/indy environment: - INDY_NODE_NAME=Node4 @@ -136,48 +136,61 @@ services: init: true container_name: controller1 environment: - - CONTAINER=node1 + - NODE_CONTAINER=node1 + - CONTROLLER_CONTAINER=controller1 volumes: - ./etc_indy:/etc/indy - $SOCK:/var/run/docker.sock restart: always - network_mode: "service:indy-node_1" + networks: + idunion_local: + ipv4_address: 10.133.133.11 + indy-controller_2: image: ${IMAGE_NAME_CONTROLLER} init: true container_name: controller2 environment: - - CONTAINER=node2 + - NODE_CONTAINER=node2 + - CONTROLLER_CONTAINER=controller2 volumes: - ./etc_indy:/etc/indy - $SOCK:/var/run/docker.sock restart: always - network_mode: "service:indy-node_2" + networks: + idunion_local: + ipv4_address: 10.133.133.12 indy-controller_3: image: ${IMAGE_NAME_CONTROLLER} init: true container_name: controller3 environment: - - CONTAINER=node3 + - NODE_CONTAINER=node3 + - CONTROLLER_CONTAINER=controller3 volumes: - ./etc_indy:/etc/indy - $SOCK:/var/run/docker.sock restart: always - network_mode: "service:indy-node_3" + networks: + idunion_local: + ipv4_address: 10.133.133.13 indy-controller_4: image: ${IMAGE_NAME_CONTROLLER} init: true container_name: controller4 environment: - - CONTAINER=node4 + - NODE_CONTAINER=node4 + - CONTROLLER_CONTAINER=controller4 volumes: - ./etc_indy:/etc/indy - $SOCK:/var/run/docker.sock restart: always - network_mode: "service:indy-node_4" + networks: + idunion_local: + ipv4_address: 10.133.133.14 networks: idunion_local: diff --git a/test/init-test-network.sh b/test/init-test-network.sh index dc40000..51137ad 100755 --- a/test/init-test-network.sh +++ b/test/init-test-network.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +NODES=${NODES:-4} if [ -z ${1+x} ]; then IMAGE_NAME_NODE=ghcr.io/idunion/indy-node-container/indy_node:buster @@ -8,7 +9,13 @@ fi echo "using image $IMAGE_NAME_NODE" -# inits 4 nodes for a local test network +# inits N nodes for a local test network mkdir -p lib_indy docker run -v "${PWD}"/etc_indy:/etc/indy -v "${PWD}"/lib_indy:/var/lib/indy "$IMAGE_NAME_NODE" \ - /bin/bash -c "rm -rf /var/lib/indy/* && generate_indy_pool_transactions --nodes 4 --clients 0 --nodeNum 1 2 3 4 --ips=\"10.133.133.1,10.133.133.2,10.133.133.3,10.133.133.4\" --network idunion_local_test && chmod -R go+w /var/lib/indy/" + /bin/bash -c "rm -rf /var/lib/indy/* && generate_indy_pool_transactions --nodes ${NODES} --clients 0 --nodeNum $(seq -s ' ' $NODES) --ips=\"$(seq -f '10.133.133.%g' -s ',' $NODES)\" --network idunion_local_test && chmod -R go+w /var/lib/indy/" + +for i in $(seq 1 $NODES); do + mkdir -p "${PWD}"/etc_indy/node$i + cp "${PWD}"/etc_indy/{indy.env,indy_config.py} "${PWD}"/etc_indy/node$i/ + echo -e "\n# node controller container IP\ncontrolServiceHost = '10.133.133.1$i'" >> "${PWD}"/etc_indy/node$i/indy_config.py +done