From 946943788a85142d1520090e0fb118db6ad846d3 Mon Sep 17 00:00:00 2001 From: marcofilho Date: Wed, 20 Nov 2024 23:20:00 +0100 Subject: [PATCH] base: add actual sha256 checking base/lnls-get-n-unpack.sh: - add shacheck function to compare expected sha256sum with actual sha256sum from file. - modify download function to iterate over each pair of arguments (arg1: url, arg2: expected sha256sum) and pass the downloaded file and expected sha256sum to shacheck. - Add description about usage in help function. Add possibility of exiting from help() with error code 1. base/install_epics.sh, base/install_opcua.sh, base/install_area_detector.sh, base/install_modules.sh, base/install_motor.sh: - Add EPICS_BASE_SHA256, $OPCUA_SHA256, etc... as arguments to be passed to lnls-get-n-unpack and install_from_github. base/install_functions.sh: - Add sha as argument to install_from_github and download_from_github. install_from_github passes sha as argument to download_from_github. base/.env: - Add all needed SHA256SUM hashes to be used by lnls-get-n-unpack. base/Dockerfile, base/musl/Dockerfile: - Add all needed SHA256SUM ARGS from .env. Add right before needed to optimize cache usage. base/docker-compose.yml, base/musl/docker-compose.yml: - Add all needed SHA256SUM arguments. --- base/.env | 24 ++++++++++++++++ base/Dockerfile | 24 ++++++++++++++++ base/docker-compose.yml | 24 ++++++++++++++++ base/install-functions.sh | 6 ++-- base/install_area_detector.sh | 5 ++-- base/install_epics.sh | 2 +- base/install_modules.sh | 53 ++++++++++++++++++++++------------- base/install_motor.sh | 2 +- base/install_opcua.sh | 2 +- base/lnls-get-n-unpack.sh | 44 +++++++++++++++++++++++++---- base/musl/Dockerfile | 20 +++++++++++++ base/musl/docker-compose.yml | 20 +++++++++++++ 12 files changed, 195 insertions(+), 31 deletions(-) diff --git a/base/.env b/base/.env index 88ef91f..756dae4 100644 --- a/base/.env +++ b/base/.env @@ -3,6 +3,8 @@ ALPINE_VERSION=3.18.6 EPICS7_BASE_VERSION=7.0.7 PVXS_VERSION=1.3.1 +EPICS7_BASE_SHA256=44d6980e19c7ad097b9e3d20c3401fb20699ed346afc307c8d1b44cf7109d475 +PVXS_SHA256=14936dda59e81a2252e1da3cf147a038cc420e4510df30c3aeb2bf1113641555 SEQUENCER_VERSION=R2-2-9 CALC_VERSION=R3-7-5 @@ -22,13 +24,35 @@ PYDEVSUP_VERSION=1.2 SNMP_VERSION=1.1.0.4 SCALER_VERSION=4.1 MCA_VERSION=R7-10 +SEQUENCER_SHA256=f5ebecdb231e106bb83db9a5fc877adb03bfd119e879a3668fdfc33d0aacb397 +CALC_SHA256=5cf1a7b3d444e763eb96ca5b9cdbcb9c29f5a6f9ac2b8d9cdb17a007d3fa8347 +ASYN_SHA256=47e993aeb300c597fcb0c3df6d3c88b9dd9e9fb90600da84cb2d2dc5b59a31aa +STREAMDEVICE_SHA256=e0640f00cd23ddd6015091d4b4e8e43a21d9e9e31d9639a5d0da6b187f42eb79 +BUSY_SHA256=1a09675bb69cdb09157b06d7276c4e4a9db8ca7e257108529e380c55452e4a53 +AUTOSAVE_SHA256=766dd7a8f71529f48d8122a3655f7986004b62ba589342153fa5a47f59119903 +SSCAN_SHA256=6911e114b07b3c200db781750035237d7dd494130f360e278df950a8358d6d78 +RECCASTER_SHA256=7108963bfa6c74d9571fe808ffa4312c1d562b30f575fb6b102fb20bc910aeb8 +IPAC_SHA256=4bd404eb9a205a32e6e25730115f6a1ab201dac69454136d4b570f880843f726 +CAPUTLOG_SHA256=6b85137906ed44a1f15358ab32fa9b4c450b37c495c65410606ea660ef4feb4c +RETOOLS_SHA256=55a3a1bfabe3898636cfa55b15df3bb80b2c526b53de7d35c8105dc7a45e1b2e +ETHER_IP_SHA256=ccf441ab842c8d24cea5b912b210852a4dc2fc005391a64f14de2ed586644155 +IOCSTATS_SHA256=13fbca066bdb34f1d84641a1b3f9fc505729866963f1e7d6d66dbc55b32a2cc8 +IPMICOMM_SHA256=7e526601461d7222834219a0f081a0550932b340eb945e515ffae94febe429f8 +PYDEVSUP_SHA256=3280cf3c4e9355f34841bc5ec38d0e5337783ea1678f31f7559b723ba8f4489b +SNMP_SHA256=f190b807aecd7d319e58263bca2ff883f891496793b68c871ada48a192d695a3 +SCALER_SHA256=faad6df4a71922ad6dcbd2d73020fad301ecd32e6e4e31ca0b0e4fd013c3bbce +MCA_SHA256=dddee716247e97e61f2e5a4bad07966ee00a7c107aed6f2d697b27337a44b9bc AREA_DETECTOR_VERSION=R3-12-1 NDSSCPIMEGA_VERSION=1.0.0 LIBSSCPIMEGA_VERSION=fb8acf533a7c01b5266bf32d60d1a5f923e19523 +LIBSSCPIMEGA_SHA256=8bb043b63a1b7bf81b3d27b8f947a134b26c9280607f3bf32f1f2ecf8e58d384 +NDSSCPIMEGA_SHA256=a688e1f54ce184fad4b0ea0d781fd69f744636fff6bddcbe477bf22087f907b0 MOTOR_VERSION=R7-3-1 PIGCS2_VERSION=60af8bdb17c1717e4545d8170f820e358ce31458 PMAC_VERSION=2-6-4b3 +PMAC_SHA256=baac82f617ddd7fb10e8967799b95353bdca4c0fefbe784636766229c1872408 OPCUA_VERSION=0.9.4 +OPCUA_SHA256=d0a947894f81c3f6a6de1adc332f19f45d50ec8c005c62f9ad0b8cacb12141af diff --git a/base/Dockerfile b/base/Dockerfile index a63d6b1..ef0f5e4 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -39,6 +39,7 @@ WORKDIR /opt/epics COPY install-functions.sh . ARG EPICS_BASE_VERSION +ARG EPICS_BASE_SHA256 ENV EPICS_BASE_PATH /opt/epics/base ENV EPICS_MODULES_PATH /opt/epics/modules ENV EPICS_RELEASE_FILE /opt/epics/RELEASE @@ -68,6 +69,25 @@ ARG PYDEVSUP_VERSION ARG SNMP_VERSION ARG SCALER_VERSION ARG MCA_VERSION +ARG PVXS_SHA256 +ARG SEQUENCER_SHA256 +ARG CALC_SHA256 +ARG ASYN_SHA256 +ARG STREAMDEVICE_SHA256 +ARG BUSY_SHA256 +ARG AUTOSAVE_SHA256 +ARG SSCAN_SHA256 +ARG RECCASTER_SHA256 +ARG IPAC_SHA256 +ARG CAPUTLOG_SHA256 +ARG RETOOLS_SHA256 +ARG ETHER_IP_SHA256 +ARG IOCSTATS_SHA256 +ARG IPMICOMM_SHA256 +ARG PYDEVSUP_SHA256 +ARG SNMP_SHA256 +ARG SCALER_SHA256 +ARG MCA_SHA256 COPY backport-ipmicomm.patch . COPY caputlog-waveform-fix.patch . @@ -77,6 +97,8 @@ RUN ./install_modules.sh ARG AREA_DETECTOR_VERSION ARG NDSSCPIMEGA_VERSION ARG LIBSSCPIMEGA_VERSION +ARG NDSSCPIMEGA_SHA256 +ARG LIBSSCPIMEGA_SHA256 COPY backport-adsupport-nanohttp.patch . COPY install_area_detector.sh . @@ -85,12 +107,14 @@ RUN ./install_area_detector.sh ARG MOTOR_VERSION ARG PIGCS2_VERSION ARG PMAC_VERSION +ARG PMAC_SHA256 COPY install_motor.sh . RUN ./install_motor.sh ARG DEBIAN_VERSION ARG OPCUA_VERSION +ARG OPCUA_SHA256 COPY install_opcua.sh . RUN ./install_opcua.sh diff --git a/base/docker-compose.yml b/base/docker-compose.yml index 1b3e065..7532139 100644 --- a/base/docker-compose.yml +++ b/base/docker-compose.yml @@ -38,3 +38,27 @@ services: PIGCS2_VERSION: ${PIGCS2_VERSION} PMAC_VERSION: ${PMAC_VERSION} OPCUA_VERSION: ${OPCUA_VERSION} + EPICS_BASE_SHA256: ${EPICS7_BASE_SHA256} + PVXS_SHA256: ${PVXS_SHA256} + SEQUENCER_SHA256: ${SEQUENCER_SHA256} + CALC_SHA256: ${CALC_SHA256} + ASYN_SHA256: ${ASYN_SHA256} + STREAMDEVICE_SHA256: ${STREAMDEVICE_SHA256} + BUSY_SHA256: ${BUSY_SHA256} + AUTOSAVE_SHA256: ${AUTOSAVE_SHA256} + SSCAN_SHA256: ${SSCAN_SHA256} + RECCASTER_SHA256: ${RECCASTER_SHA256} + IPAC_SHA256: ${IPAC_SHA256} + CAPUTLOG_SHA256: ${CAPUTLOG_SHA256} + RETOOLS_SHA256: ${RETOOLS_SHA256} + ETHER_IP_SHA256: ${ETHER_IP_SHA256} + IOCSTATS_SHA256: ${IOCSTATS_SHA256} + IPMICOMM_SHA256: ${IPMICOMM_SHA256} + PYDEVSUP_SHA256: ${PYDEVSUP_SHA256} + SNMP_SHA256: ${SNMP_SHA256} + SCALER_SHA256: ${SCALER_SHA256} + MCA_SHA256: ${MCA_SHA256} + NDSSCPIMEGA_SHA256: ${NDSSCPIMEGA_SHA256} + LIBSSCPIMEGA_SHA256: ${LIBSSCPIMEGA_SHA256} + PMAC_SHA256: ${PMAC_SHA256} + OPCUA_SHA256: ${OPCUA_SHA256} diff --git a/base/install-functions.sh b/base/install-functions.sh index 4d5855a..87af52a 100644 --- a/base/install-functions.sh +++ b/base/install-functions.sh @@ -10,8 +10,9 @@ download_from_github() { github_org=$1 module_name=$2 commit=$3 + sha=$4 - lnls-get-n-unpack -l https://github.com/$github_org/$module_name/archive/$commit.tar.gz + lnls-get-n-unpack -l https://github.com/$github_org/$module_name/archive/$commit.tar.gz $sha mv $module_name-$commit $module_name } @@ -59,7 +60,8 @@ install_from_github() { dependency_name=$3 tag=$4 release_content="$5" + sha=$6 - download_from_github $github_org $module_name $tag + download_from_github $github_org $module_name $tag $sha install_module $flag_ioc $module_name $dependency_name "$release_content" } diff --git a/base/install_area_detector.sh b/base/install_area_detector.sh index c995cef..cecccea 100755 --- a/base/install_area_detector.sh +++ b/base/install_area_detector.sh @@ -99,7 +99,7 @@ make clean cd .. -download_from_github cnpem ssc-pimega $LIBSSCPIMEGA_VERSION +download_from_github cnpem ssc-pimega $LIBSSCPIMEGA_VERSION $LIBSSCPIMEGA_SHA256 make -C ssc-pimega/c install install_from_github cnpem NDSSCPimega NDSSCPIMEGA $NDSSCPIMEGA_VERSION " @@ -107,4 +107,5 @@ EPICS_BASE ASYN AREA_DETECTOR ADCORE -" +" \ +$NDSSCPIMEGA_SHA256 diff --git a/base/install_epics.sh b/base/install_epics.sh index 498115c..c906f8c 100755 --- a/base/install_epics.sh +++ b/base/install_epics.sh @@ -4,7 +4,7 @@ set -ex . /opt/epics/install-functions.sh -lnls-get-n-unpack -l https://epics-controls.org/download/base/base-${EPICS_BASE_VERSION}.tar.gz +lnls-get-n-unpack -l https://epics-controls.org/download/base/base-${EPICS_BASE_VERSION}.tar.gz $EPICS_BASE_SHA256 mv base-${EPICS_BASE_VERSION} ${EPICS_BASE_PATH} patch -d ${EPICS_BASE_PATH} -Np1 < backport-epics-base-musl.patch diff --git a/base/install_modules.sh b/base/install_modules.sh index 999d168..961d469 100755 --- a/base/install_modules.sh +++ b/base/install_modules.sh @@ -6,52 +6,61 @@ set -ex install_from_github mdavidsaver pvxs PVXS $PVXS_VERSION " EPICS_BASE -" +" \ +$PVXS_SHA256 install_from_github epics-modules sequencer SNCSEQ $SEQUENCER_VERSION " EPICS_BASE -" +" \ +$SEQUENCER_SHA256 install_from_github epics-modules calc CALC $CALC_VERSION " EPICS_BASE -" +" \ +$CALC_SHA256 # Build asyn without seq since it's only needed for testIPServer install_from_github epics-modules asyn ASYN $ASYN_VERSION " EPICS_BASE CALC -" +" \ +$ASYN_SHA256 install_from_github paulscherrerinstitute StreamDevice STREAM $STREAMDEVICE_VERSION " EPICS_BASE ASYN CALC -" +" \ +$STREAMDEVICE_SHA256 install_from_github epics-modules busy BUSY $BUSY_VERSION " EPICS_BASE ASYN -" +" \ +$BUSY_SHA256 install_from_github epics-modules autosave AUTOSAVE $AUTOSAVE_VERSION " EPICS_BASE -" +" \ +$AUTOSAVE_SHA256 install_from_github epics-modules sscan SSCAN $SSCAN_VERSION " EPICS_BASE SNCSEQ -" +" \ +$SSCAN_SHA256 -download_from_github ChannelFinder recsync $RECCASTER_VERSION +download_from_github ChannelFinder recsync $RECCASTER_VERSION $RECCASTER_SHA256 install_module recsync/client RECCASTER " EPICS_BASE " install_from_github epics-modules ipac IPAC $IPAC_VERSION " EPICS_BASE -" +" \ +$IPAC_SHA256 -download_from_github epics-modules caPutLog $CAPUTLOG_VERSION +download_from_github epics-modules caPutLog $CAPUTLOG_VERSION $CAPUTLOG_SHA256 patch -d caPutLog -Np1 < caputlog-waveform-fix.patch install_module caPutLog CAPUTLOG " EPICS_BASE @@ -59,24 +68,27 @@ EPICS_BASE install_from_github brunoseivam retools RETOOLS $RETOOLS_VERSION " EPICS_BASE -" +" \ +$RETOOLS_SHA256 install_from_github -i epics-modules ether_ip ETHER_IP $ETHER_IP_VERSION " EPICS_BASE -" +" \ +$ETHER_IP_SHA256 install_from_github epics-modules iocStats DEVIOCSTATS $IOCSTATS_VERSION " EPICS_BASE -" +" \ +$IOCSTATS_SHA256 -download_from_github slac-epics-modules ipmiComm $IPMICOMM_VERSION +download_from_github slac-epics-modules ipmiComm $IPMICOMM_VERSION $IPMICOMM_SHA256 patch -d ipmiComm -Np1 < backport-ipmicomm.patch JOBS=1 install_module ipmiComm IPMICOMM " EPICS_BASE ASYN " -download_from_github mdavidsaver pyDevSup $PYDEVSUP_VERSION +download_from_github mdavidsaver pyDevSup $PYDEVSUP_VERSION $PYDEVSUP_SHA256 echo PYTHON=python3 >> pyDevSup/configure/CONFIG_SITE install_module pyDevSup PYDEVSUP " EPICS_BASE @@ -84,7 +96,8 @@ EPICS_BASE mkdir snmp cd snmp -lnls-get-n-unpack -l https://groups.nscl.msu.edu/controls/files/epics-snmp-$SNMP_VERSION.zip +lnls-get-n-unpack -l https://groups.nscl.msu.edu/controls/files/epics-snmp-$SNMP_VERSION.zip \ +$SNMP_SHA256 cd .. install_module -i snmp SNMP " EPICS_BASE @@ -93,7 +106,8 @@ EPICS_BASE install_from_github epics-modules scaler SCALER $SCALER_VERSION " EPICS_BASE ASYN -" +" \ +$SCALER_SHA256 install_from_github -i epics-modules mca MCA $MCA_VERSION " EPICS_BASE @@ -105,4 +119,5 @@ SNCSEQ AUTOSAVE ASYN MCA -" +" \ +$MCA_SHA256 diff --git a/base/install_motor.sh b/base/install_motor.sh index f0a2582..9e746a1 100755 --- a/base/install_motor.sh +++ b/base/install_motor.sh @@ -52,7 +52,7 @@ SNCSEQ cd $EPICS_MODULES_PATH -download_from_github dls-controls pmac $PMAC_VERSION +download_from_github dls-controls pmac $PMAC_VERSION $PMAC_SHA256 rm pmac/configure/RELEASE.local.linux-x86_64 rm pmac/configure/RELEASE.linux-x86_64.Common diff --git a/base/install_opcua.sh b/base/install_opcua.sh index e61890a..a83f1da 100755 --- a/base/install_opcua.sh +++ b/base/install_opcua.sh @@ -6,7 +6,7 @@ set -ex opcua_release_url=https://github.com/epics-modules/opcua/releases/download/v${OPCUA_VERSION} opcua_release_file=IOC_opcua-${OPCUA_VERSION}_Base-${EPICS_BASE_VERSION}_debian${DEBIAN_VERSION%.*}.tar.gz -lnls-get-n-unpack -l $opcua_release_url/$opcua_release_file +lnls-get-n-unpack -l $opcua_release_url/$opcua_release_file $OPCUA_SHA256 mv binaryOpcuaIoc opcua install_module -i opcua OPCUA " diff --git a/base/lnls-get-n-unpack.sh b/base/lnls-get-n-unpack.sh index bfcc517..0a0bb08 100755 --- a/base/lnls-get-n-unpack.sh +++ b/base/lnls-get-n-unpack.sh @@ -6,29 +6,32 @@ set -eu help () { echo "lnls-get-n-unpack: download and extract archive from the network" - echo -e "Usage: lnls-get-n-unpack [URL2] [...] [URLN]\n" + echo -e "Usage: lnls-get-n-unpack <1SHA256SUM> [URL2] [2SHA256SUM] [...] [URLN] [NSHA256SUM]\n" echo "Extraction mode:" echo " -l extracts to local directory (./)" echo " -r extracts to root directory (/)" echo "URL:" echo " url to download source from." - exit 0; + echo "SHA256SUM:" + echo " Reference sha256 hash to compare url download with." + exit $1; } check_arguments() { # No arguments = call help and exit. if [ -z ${1+nothing} ]; then - help + help 0 fi # if -h is anywhere arguments list, call help and exit. for arg in "$@"; do if [ "$arg" == "-h" ] || [ "$arg" == "--help" ]; then - help + help 0 fi done + # Check extraction mode case "$1" in -r) dest=/ ;; -l) dest=. ;; @@ -37,19 +40,48 @@ check_arguments() { ;; esac + # Check if we have odd number of arguments (extraction_mode + N*url + N*sha256sum) + if [ $(( $# % 2 )) -ne 1 ]; then + >&2 echo "ERROR: Even number of arguments detected. Something is wrong." + help 1 + fi + +} + +shacheck() { + + download_dir=$1 + sha=$2 + url=$3 + + downloaded_file=$(find $download_dir -type f) + if [[ $(echo $downloaded_file | wc -w) -ne 1 ]]; then + echo "ERROR: Download of $url is yielding something different than one single file." + echo "Don't know how to proceed. Exiting..." + exit 1 + fi + + if ! echo $sha $downloaded_file | sha256sum -c; then + echo "ERROR: SHA $sha for URL $url does not match." + exit 1 + fi + } download () { shift # Throw extraction mode argument away - for url; do + while [[ $# -gt 1 ]]; do + url=$1 + sha=$2 download_dir=$(mktemp -d) echo Downloading "$url"... wget -P $download_dir "$url" &> /tmp/wget.log || (cat /tmp/wget.log && false) filename=$(basename $download_dir/*) + shacheck $download_dir $sha $url if [[ ${filename,,} == *".zip" ]]; then unzip -qo $download_dir/$filename -d $dest @@ -58,6 +90,8 @@ download () { fi rm -rf $download_dir /tmp/wget.log + + shift 2 done } diff --git a/base/musl/Dockerfile b/base/musl/Dockerfile index 826bf81..04cc339 100644 --- a/base/musl/Dockerfile +++ b/base/musl/Dockerfile @@ -34,6 +34,7 @@ COPY lnls-get-n-unpack.sh /usr/local/bin/lnls-get-n-unpack COPY lnls-run.sh /usr/local/bin/lnls-run ARG EPICS_BASE_VERSION +ARG EPICS_BASE_SHA256 ENV EPICS_BASE_PATH /opt/epics/base ENV EPICS_MODULES_PATH /opt/epics/modules ENV EPICS_RELEASE_FILE /opt/epics/RELEASE @@ -64,6 +65,25 @@ ARG PYDEVSUP_VERSION ARG SNMP_VERSION ARG SCALER_VERSION ARG MCA_VERSION +ARG PVXS_SHA256 +ARG SEQUENCER_SHA256 +ARG CALC_SHA256 +ARG ASYN_SHA256 +ARG STREAMDEVICE_SHA256 +ARG BUSY_SHA256 +ARG AUTOSAVE_SHA256 +ARG SSCAN_SHA256 +ARG RECCASTER_SHA256 +ARG IPAC_SHA256 +ARG CAPUTLOG_SHA256 +ARG RETOOLS_SHA256 +ARG ETHER_IP_SHA256 +ARG IOCSTATS_SHA256 +ARG IPMICOMM_SHA256 +ARG PYDEVSUP_SHA256 +ARG SNMP_SHA256 +ARG SCALER_SHA256 +ARG MCA_SHA256 WORKDIR ${EPICS_MODULES_PATH} COPY backport-ipmicomm.patch . diff --git a/base/musl/docker-compose.yml b/base/musl/docker-compose.yml index 58bd435..be64752 100644 --- a/base/musl/docker-compose.yml +++ b/base/musl/docker-compose.yml @@ -31,3 +31,23 @@ services: SNMP_VERSION: ${SNMP_VERSION} SCALER_VERSION: ${SCALER_VERSION} MCA_VERSION: ${MCA_VERSION} + EPICS_BASE_SHA256: ${EPICS7_BASE_SHA256} + PVXS_SHA256: ${PVXS_SHA256} + SEQUENCER_SHA256: ${SEQUENCER_SHA256} + CALC_SHA256: ${CALC_SHA256} + ASYN_SHA256: ${ASYN_SHA256} + STREAMDEVICE_SHA256: ${STREAMDEVICE_SHA256} + BUSY_SHA256: ${BUSY_SHA256} + AUTOSAVE_SHA256: ${AUTOSAVE_SHA256} + SSCAN_SHA256: ${SSCAN_SHA256} + RECCASTER_SHA256: ${RECCASTER_SHA256} + IPAC_SHA256: ${IPAC_SHA256} + CAPUTLOG_SHA256: ${CAPUTLOG_SHA256} + RETOOLS_SHA256: ${RETOOLS_SHA256} + ETHER_IP_SHA256: ${ETHER_IP_SHA256} + IOCSTATS_SHA256: ${IOCSTATS_SHA256} + IPMICOMM_SHA256: ${IPMICOMM_SHA256} + PYDEVSUP_SHA256: ${PYDEVSUP_SHA256} + SNMP_SHA256: ${SNMP_SHA256} + SCALER_SHA256: ${SCALER_SHA256} + MCA_SHA256: ${MCA_SHA256}