Clean up dockerfiles (#124)

* clean up ngen-deps Dockerfile * more clean up * remove unnecessary declarnation * remove unused args * more clean up * use single intermediate image for downloads
CIROH-UA · Mar 5, 2024 · ff7d05d · ff7d05d
1 parent 8411887
commit ff7d05d
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 205 deletions.
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -6,10 +6,6 @@ FROM awiciroh/ngen:${TAG_NAME} as rocky_build_staging
 
 ARG DATASET_DIRECTORIES
 
-#RUN cd ./ngen && mkdir ${WORKDIR}/bin && cp cmake_build/ngen ${WORKDIR}/bin && cp -r data ${WORKDIR}/data \
-#    && cd $WORKDIR && rm -rf ngen boost
-
-
 # Remove the boost headers now that ngen is compiled; also update path and make sure dataset directory is there
 RUN rm -rf ${BOOST_ROOT} && echo "export PATH=${PATH}" >> /etc/profile \
     && mkdir -p /dmod/datasets && chown root /dmod/datasets \

diff --git a/docker/Dockerfile.ngen b/docker/Dockerfile.ngen
@@ -1,13 +1,11 @@
 ################################################################################################################
 ##### External Global Args Declaration and Defaults
 ################################################################################################################
-ARG NGEN_BUILD_CONFIG_TYPE="Release" \
-    REPO_URL=https://github.com/NOAA-OWP/ngen.git \
+ARG REPO_URL=https://github.com/NOAA-OWP/ngen.git \
     BRANCH=master \
     COMMIT \
     TAG_NAME 
 
-
 ################################################################################################################
 ################################################################################################################
 ##### Create intermediate Docker build stage for building framework in Rocky Linux environment
@@ -39,11 +37,7 @@ FROM awiciroh/t-route:${TAG_NAME} as troute
 ##### Create intermediate Docker build stage for building framework in Rocky Linux environment
 FROM awiciroh/ngen-deps:${TAG_NAME} as rocky_build_ngen
 
-ARG REPO_URL \
-    BRANCH \
-    COMMIT \
-    NGEN_BUILD_CONFIG_TYPE="Release" \
-    NGEN_ACTIVATE_C="ON" \
+ARG NGEN_ACTIVATE_C="ON" \
     NGEN_ACTIVATE_FORTRAN="ON" \
     NGEN_ACTIVATE_PYTHON="ON" \
     NGEN_NETCDF_ACTIVE="ON" \
@@ -52,13 +46,7 @@ ARG REPO_URL \
     NGEN_UDUNITS_ACTIVE="ON" \
     NGEN_UDUNITS_QUIET="ON" \
     BUILD_NGEN_SERIAL="true" \
-    BUILD_NGEN_PARALLEL="true" \
-    BUILD_NGEN_PARTITIONER="true" \
-    BUILD_NOAH_OWP="true" \
-    BUILD_CFE="true" \
-    BUILD_TOPMODEL="true" \
-    BUILD_PET="true" \
-    BUILD_SLOTH="true"
+    BUILD_NGEN_PARALLEL="true"
 
 COPY --chown=${USER} --from=rocky_init_repo /ngen/ngen /ngen/ngen
 COPY --chown=${USER} --from=troute /ngen/t-route/wheels /tmp/t-route-wheels
@@ -76,25 +64,6 @@ RUN if [ "${NGEN_ACTIVATE_PYTHON}" == "ON" ]; then \
     fi
 USER ${USER}
 
-#These patches are from ngen commit 7551590a415b89026559c1c570d4154e4746161b
-#and are needed to fix an I/O bug in the sub modules (and bmi tests) that cause an infinite loop
-#https://github.com/NOAA-OWP/ngen/pull/515, https://github.com/NOAA-OWP/ngen/pull/518 both merged upstream, patch no longer needed
-# All the affected submodules have been patched and the ngen submodule commits have been updated
-# so this patch is no longer needed.  I'm leaving these lines commented out in case custom patches
-# need to be applied in the future.
-# Patches can be generated with
-# generate submodule patch:
-#git submodule --quiet foreach --recursive 'export NAME="${PWD##*/}"; git --no-pager diff --src-prefix="a/extern/${NAME}/${NAME}/" --dst-prefix="b/extern/${NAME}/${NAME}/"' > fix_io_sub.patch
-# generate patch of ngen without submodules
-#git --no-pager diff --ignore-submodules > fix_io.patch
-#It is then probably a good idea to to add a commit hash to thee patch file name as below
-#to indicate from what point the patch was generated.
-
-#COPY fix_io_sub_7551590a415b89026559c1c570d4154e4746161b.patch ${WORKDIR}/ngen/fix_io_sub.patch
-# Apply the IO fix to submodules, once they all get patched/merged, this can be dropped...
-#RUN cd ${WORKDIR}/ngen && git apply --reject --whitespace=fix \
-#    #patch the submodules
-#    fix_io_sub.patch
 WORKDIR /ngen/ngen
 RUN if [ "${NGEN_ACTIVATE_PYTHON}" == "ON" ]; then \
     pip3 install -r extern/test_bmi_py/requirements.txt; \

diff --git a/docker/Dockerfile.ngen-deps b/docker/Dockerfile.ngen-deps
@@ -4,192 +4,72 @@
 # Any args that will need to be set, first declare them here, then later redeclare them within stage that uses.
 # Also, for those with defaults, set those up here so that's easier to find.
 
-# Swap or override this as needed (below are the "standard" types from
-#   https://cmake.org/cmake/help/latest/manual/cmake-buildsystem.7.html#default-and-custom-configurations)
-ARG DOCKER_INTERNAL_REGISTRY \
-    NGEN_BUILD_CONFIG_TYPE="Release" \
-    DATASET_DIRECTORIES="config forcing hydrofabric observation output" 
-ARG REPO_URL=https://github.com/NOAA-OWP/ngen.git \
-    BRANCH=master \
-    WORKDIR=/ngen \
-    TROUTE_REPO_URL=https://github.com/NOAA-OWP/t-route.git \
-    TROUTE_BRANCH=master \
-#### Default arguments for required dependencies needed during various build stages
-# The Rocky-Linux-based "ngen-deps" stage, rocky-ngen-deps
-# TODO: later look at separating build and run images again, and install static lib netcdf packages in run images
-#ARG ROCKY_NGEN_DEPS_REQUIRED="mpich mpich-devel sudo gcc gcc-c++ make cmake tar git gcc-gfortran libgfortran \
-#    python39 python39-devel python39-pip python39-numpy \
-#    netcdf-cxx4-devel netcdf-cxx4-mpich-devel netcdf-fortran-devel netcdf-fortran-mpich-devel \
-#    hdf5 hdf5-devel hdf5-mpich hdf5-mpich-devel \
-#    bzip2 expat expat-devel flex bison udunits2 udunits2-devel"
+ARG WORKDIR=/ngen \
     ROCKY_NGEN_DEPS_REQUIRED="sudo gcc gcc-c++ make cmake tar git gcc-gfortran libgfortran sqlite sqlite-devel \
     python3 python3-devel python3-pip gdal gdal-devel\
     bzip2 expat expat-devel flex bison udunits2 udunits2-devel zlib-devel" \
-# TODO: removed texinfo from list because it couldn't be found; make sure this doesn't lead to issues
     BOOST_VERSION=1.79.0 \
-#TODO Try mpich 4.x for native arm build support (https://raw.githubusercontent.com/pmodels/mpich/v4.1.1/CHANGES)
-#mpich 3.2 doesn't work well gfortran 11 it seems, an alignment error crops up, but 3.3.2 seems to work...
     MPICH_VERSION="3.3.2" \
-    MIN_PYTHON="3.8.0" \
-    MIN_NUMPY="1.18.0" \
     NETCDF_C_VERSION=4.8.1 \
     NETCDF_CXX_VERSION=4.3.1 \
     NETCDF_FORTRAN_VERSION=4.6.0 \
     HD5_VERSION=1.10.9 \
     BLOSC2_VERSION=v2.2.0 \
-    FCFLAGS="-w -fallow-argument-mismatch -O2" \
-    FFLAGS="-w -fallow-argument-mismatch -O2" \
     MPICH_CONFIGURE_OPTIONS="" \
     MPICH_MAKE_OPTIONS \
-    NGEN_ACTIVATE_C="ON" \
-    NGEN_ACTIVATE_FORTRAN="ON" \
-    NGEN_ACTIVATE_PYTHON="ON" \
-    NGEN_NETCDF_ACTIVE="ON" \
-    NGEN_ROUTING_ACTIVE="ON" \
-    NGEN_UDUNITS_ACTIVE="ON" \
-    NGEN_UDUNITS_QUIET="ON" \
-    BUILD_NGEN_SERIAL="true" \
-    BUILD_NGEN_PARALLEL="true" \
-    BUILD_NGEN_PARTITIONER="true" \
-    BUILD_NOAH_OWP="true" \
-    BUILD_CFE="true" \
-    BUILD_TOPMODEL="true" \
-    BUILD_PET="true" \
-    BUILD_SLOTH="true" 
+    NGEN_ACTIVATE_PYTHON="ON"
 
 ################################################################################################################
 ################################################################################################################
 ##### Create intermediate Docker build stage for Rocky-Linux-based "base"
 FROM rockylinux:9.1 as rocky-base
 
-#USER=root
-
-ARG ROCKY_BASE_REQUIRED="sudo bash git" \
-    WORKDIR=/ngen 
+ARG WORKDIR=/ngen 
 RUN dnf update -y \
     && dnf install -y 'dnf-command(config-manager)' \
     && dnf config-manager --set-enabled crb \
     && dnf install -y epel-release \
-    && dnf -y install ${ROCKY_BASE_REQUIRED} 
-    # Note that adduser -p expects an encrypted/hashed password, so it will ignore a simple password \
-    # Config SSH Daemon \
-    # Unlock non-password USER to enable SSH login \
-    #&& passwd -u ${USER} 
-
-
-
-
-# Switch back to default user when continue the build process
+    && dnf -y install sudo bash git
 
 ################################################################################################################
 ################################################################################################################
-##### Create intermediate Docker build stage for downloading Boost
-FROM rockylinux:9.1 AS download_boost
+##### Create intermediate Docker build stage for downloading netcdf-fortran
+FROM rocky-base AS downloads
 
 # Redeclaring inside this stage to get default from before first FROM
-ARG BOOST_VERSION
+ARG BOOST_VERSION \
+    MPICH_VERSION \
+    HD5_VERSION \
+    NETCDF_C_VERSION \
+    NETCDF_FORTRAN_VERSION \
+    NETCDF_CXX_VERSION
 
 RUN curl -L -o boost_${BOOST_VERSION//./_}.tar.bz2 https://sourceforge.net/projects/boost/files/boost/${BOOST_VERSION}/boost_${BOOST_VERSION//./_}.tar.bz2/download \
     && mkdir /boost \
     && mv boost_${BOOST_VERSION//./_}.tar.bz2 /boost/.
 
-################################################################################################################
-##https://support.hdfgroup.org/ftp/HDF5/releases##############################################################################################################
-##### Create intermediate Docker build stage for downloading MPICH
-FROM rockylinux:9.1 AS download_mpich
-
-# Redeclaring inside this stage to get default from before first FROM
-ARG MPICH_VERSION
-
 RUN curl -o /tmp/mpich-${MPICH_VERSION}.tar.gz https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz
 
-
-################################################################################################################
-################################################################################################################
-##### Create intermediate Docker build stage for downloading MPICH
-FROM rockylinux:9.1 AS download_hd5
-
-# Redeclaring inside this stage to get default from before first FROM
-ARG HD5_VERSION
-
 RUN curl -o /tmp/hdf5-${HD5_VERSION}.tar.gz https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-${HD5_VERSION}/src/hdf5-${HD5_VERSION}.tar.gz
 
-################################################################################################################
-################################################################################################################
-##### Create intermediate Docker build stage for downloading netcdf
-FROM rocky-base AS download_netcdf
+RUN curl -s https://api.github.com/repos/Unidata/netcdf-c/releases \
+    | grep "tarball_url" \
+    | grep "${NETCDF_C_VERSION}" \
+    | cut -d '"' -f 4 \
+    | xargs curl -L -o /tmp/netcdf-${NETCDF_C_VERSION}.tar.gz
 
-# Redeclaring inside this stage to get default from before first FROM
-ARG NETCDF_C_VERSION
+RUN curl -s https://api.github.com/repos/Unidata/netcdf-fortran/releases \
+    | grep "tarball_url" \
+    | grep "${NETCDF_FORTRAN_VERSION}" \
+    | cut -d '"' -f 4 \
+    | xargs curl -L -o /tmp/netcdf-fortran-${NETCDF_FORTRAN_VERSION}.tar.gz
 
-RUN if [[ "${NETCDF_C_VERSION}" == "latest" ]]; then \
-            curl -s https://api.github.com/repos/Unidata/netcdf-c/releases/latest \
-                | grep "tarball_url" \
-                | cut -d '"' -f 4 \
-                | xargs curl -L -o /tmp/netcdf-${NETCDF_C_VERSION}.tar.gz ; \
-        else \
-            curl -s https://api.github.com/repos/Unidata/netcdf-c/releases \
-                | grep "tarball_url" \
-                | grep "${NETCDF_C_VERSION}" \
-                | cut -d '"' -f 4 \
-                | xargs curl -L -o /tmp/netcdf-${NETCDF_C_VERSION}.tar.gz ; \
-        #fi \
-        #&& mkdir /tmp/netcdf \
-        #&& tar -xzf /tmp/netcdf-${NETCDF_C_VERSION}.tar.gz -C /tmp/netcdf --strip 1 \
-        #&& rm /tmp/netcdf-${NETCDF_C_VERSION}.tar.gz
-        fi
-
-################################################################################################################
-################################################################################################################
-##### Create intermediate Docker build stage for downloading netcdf-fortran
-FROM rocky-base AS download_netcdf_fortran
-
-# Redeclaring inside this stage to get default from before first FROM
-ARG NETCDF_FORTRAN_VERSION
-
-RUN if [[ "${NETCDF_FORTRAN_VERSION}" == "latest" ]]; then \
-            curl -s https://api.github.com/repos/Unidata/netcdf-fortran/releases/latest \
-                | grep "tarball_url" \
-                | cut -d '"' -f 4 \
-                | xargs curl -L -o /tmp/netcdf-fortran-${NETCDF_FORTRAN_VERSION}.tar.gz ; \
-        else \
-            curl -s https://api.github.com/repos/Unidata/netcdf-fortran/releases \
-                | grep "tarball_url" \
-                | grep "${NETCDF_FORTRAN_VERSION}" \
-                | cut -d '"' -f 4 \
-                | xargs curl -L -o /tmp/netcdf-fortran-${NETCDF_FORTRAN_VERSION}.tar.gz ; \
-        #fi \
-        #&& mkdir /tmp/netcdf-fortran \
-        #&& tar -xzf /tmp/netcdf-fortran-${NETCDF_FORTRAN_VERSION}.tar.gz -C /tmp/netcdf-fortran --strip 1 \
-        #&& rm /tmp/netcdf-fortran-${NETCDF_FORTRAN_VERSION}.tar.gz
-        fi
-
-################################################################################################################
-################################################################################################################
-##### Create intermediate Docker build stage for downloading NetCDF C++
-FROM rocky-base AS download_netcdf_cxx
-
-# Redeclaring inside this stage to get default from before first FROM
-ARG NETCDF_CXX_VERSION
+RUN curl -s https://api.github.com/repos/Unidata/netcdf-cxx4/releases \
+    | grep "tarball_url" \
+    | grep "${NETCDF_CXX_VERSION}" \
+    | cut -d '"' -f 4 \
+    | xargs curl -L -o /tmp/netcdf-cxx4-${NETCDF_CXX_VERSION}.tar.gz
 
-RUN if [[ "${NETCDF_CXX_VERSION}" == "latest" ]]; then \
-            curl -s https://api.github.com/repos/Unidata/netcdf-cxx4/releases/latest \
-                | grep "tarball_url" \
-                | cut -d '"' -f 4 \
-                | xargs curl -L -o /tmp/netcdf-cxx4-${NETCDF_CXX_VERSION}.tar.gz ; \
-        else \
-            curl -s https://api.github.com/repos/Unidata/netcdf-cxx4/releases \
-                | grep "tarball_url" \
-                | grep "${NETCDF_CXX_VERSION}" \
-                | cut -d '"' -f 4 \
-                | xargs curl -L -o /tmp/netcdf-cxx4-${NETCDF_CXX_VERSION}.tar.gz ; \
-        #fi \
-        #&& mkdir /tmp/netcdf-cxx4 \
-        #&& tar -xzf /tmp/netcdf-cxx4-${NETCDF_CXX_VERSION}.tar.gz -C /tmp/netcdf-cxx4 --strip 1 \
-        #&& rm /tmp/netcdf-cxx4-${NETCDF_CXX_VERSION}.tar.gz
-        fi
-    # This URL might also work for the analogous version:
-    #   https://downloads.unidata.ucar.edu/netcdf-cxx/4.3.1/netcdf-cxx4-4.3.1.tar.gz
 ################################################################################################################
 ################################################################################################################
 ##### Create intermediate Docker build stage for just prior to "ngen-deps" for installing packaged deps
@@ -198,23 +78,18 @@ FROM rocky-base as rocky-ngen-packaged-deps
 ARG ROCKY_NGEN_DEPS_REQUIRED
 
 # Note that this includes numpy, which is needed for Python BMI support, regardless of BMI module 
-#USER root
 RUN dnf update -y && dnf install -y ${ROCKY_NGEN_DEPS_REQUIRED} \
     && pip3 install --upgrade pip \
     && if [ "${NGEN_ACTIVATE_PYTHON}" == "ON" ]; then pip3 install numpy; fi
-#USER ${USER}
 
 ################################################################################################################
 ################################################################################################################
 ##### Create intermediate Docker build stage for Rocky-Linux-based "ngen-deps"
 FROM rocky-ngen-packaged-deps as rocky-ngen-deps
 
-#USER root
 ARG BOOST_VERSION \
     WORKDIR  \
     MPICH_VERSION \
-    MIN_PYTHON \
-    MIN_NUMPY \
     ROCKY_NGEN_DEPS_REQUIRED \
     MPICH_CONFIGURE_OPTIONS \
     MPICH_MAKE_OPTIONS \
@@ -233,12 +108,12 @@ ENV WORKDIR=${WORKDIR} \
     NETCDF_FORTRAN_VERSION=${NETCDF_FORTRAN_VERSION} \
     HYDRA_HOST_FILE=/etc/opt/hosts
 
-COPY --from=download_boost /boost ${WORKDIR}/boost 
-COPY --from=download_mpich /tmp/mpich-${MPICH_VERSION}.tar.gz /tmp/ngen-deps/mpich-${MPICH_VERSION}.tar.gz  
-COPY --from=download_hd5 /tmp/hdf5-${HD5_VERSION}.tar.gz /tmp/ngen-deps/hdf5-${HD5_VERSION}.tar.gz 
-COPY --from=download_netcdf /tmp/netcdf-${NETCDF_C_VERSION}.tar.gz /tmp/ngen-deps/netcdf-${NETCDF_C_VERSION}.tar.gz 
-COPY --from=download_netcdf_cxx /tmp/netcdf-cxx4-${NETCDF_CXX_VERSION}.tar.gz /tmp/ngen-deps/netcdf-cxx4-${NETCDF_CXX_VERSION}.tar.gz 
-COPY --from=download_netcdf_fortran /tmp/netcdf-fortran-${NETCDF_FORTRAN_VERSION}.tar.gz /tmp/ngen-deps/netcdf-fortran-${NETCDF_FORTRAN_VERSION}.tar.gz
+COPY --from=downloads /boost ${WORKDIR}/boost 
+COPY --from=downloads /tmp/mpich-${MPICH_VERSION}.tar.gz /tmp/ngen-deps/mpich-${MPICH_VERSION}.tar.gz  
+COPY --from=downloads /tmp/hdf5-${HD5_VERSION}.tar.gz /tmp/ngen-deps/hdf5-${HD5_VERSION}.tar.gz 
+COPY --from=downloads /tmp/netcdf-${NETCDF_C_VERSION}.tar.gz /tmp/ngen-deps/netcdf-${NETCDF_C_VERSION}.tar.gz 
+COPY --from=downloads /tmp/netcdf-cxx4-${NETCDF_CXX_VERSION}.tar.gz /tmp/ngen-deps/netcdf-cxx4-${NETCDF_CXX_VERSION}.tar.gz 
+COPY --from=downloads /tmp/netcdf-fortran-${NETCDF_FORTRAN_VERSION}.tar.gz /tmp/ngen-deps/netcdf-fortran-${NETCDF_FORTRAN_VERSION}.tar.gz
 
 RUN pip3 install --upgrade pip \
     && pip3 install -t /ngen numpy pandas pyyaml bmipy Cython netCDF4 wheel packaging opencv-contrib-python-headless \
@@ -268,8 +143,6 @@ RUN pip3 install --upgrade pip \
     && cd netcdf \
     && LIBS=curl && ./configure --prefix=/usr \
     && make -j $(nproc) && make install \
-    # TODO: if we run into any problem, might need to reactivate this \
-    #&& make check \
     ##### Build and install NetCDF Fortran \
     && cd /tmp/ngen-deps \
     && mkdir netcdf-fortran \
@@ -288,15 +161,13 @@ RUN pip3 install --upgrade pip \
     && ../configure --enable-parallel --prefix=/usr \
     && cmake .. \
     && make \
-    # TODO: if we run into any problem, might need to reactivate this \
-    #&& ctest \
     && make install \
     # Install required python dependency packages with Pip \
     # Except blosc2, since packaged wheels on pypi seem to have some issues, build it ourselves
     && cd /tmp/ngen-deps \
     && git clone https://github.com/Blosc/python-blosc2/ \
     && cd python-blosc2 \
-    #checkout a release tag
+    # checkout a release tag
     && git checkout ${BLOSC2_VERSION} \ 
     && git submodule update --init --recursive \
     && python3 -m pip install -r requirements-build.txt \

diff --git a/docker/Dockerfile.t-route b/docker/Dockerfile.t-route
@@ -32,10 +32,6 @@ RUN cd ${WORKDIR} \
 ARG TAG_NAME
 FROM awiciroh/ngen-deps:${TAG_NAME} as rocky_build_troute
 
-ARG REPO_URL \
-    BRANCH \
-    COMMIT
-
 COPY --chown=root --from=rocky_init_troute_repo ${WORKDIR}/t-route ${WORKDIR}/t-route
 
 RUN cp -s /usr/bin/python3 /usr/bin/python \