diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..23c4cb3b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +--- +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..e9899c17 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +--- +name: ci + +# yamllint disable-line rule:truthy +on: + push: + pull_request: + +concurrency: + # yamllint disable-line rule:line-length + group: ci-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} + cancel-in-progress: true + +jobs: + prechecks: + uses: ./.github/workflows/pre-commit.yml + all-prechecks: + needs: [prechecks] + runs-on: ubuntu-latest + steps: + - name: Success + run: "true" + tests: + needs: [prechecks] + uses: ./.github/workflows/test.yml + all: + needs: [tests] + runs-on: ubuntu-latest + steps: + - name: Success + run: "true" diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index c2a1ed45..f64a609e 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -1,16 +1,20 @@ --- -# In the absence of using the pre-commit.ci service... name: pre-commit # yamllint disable-line rule:truthy on: - pull_request: - push: + workflow_call: + workflow_dispatch: + +concurrency: + # yamllint disable-line rule:line-length + group: style-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} + cancel-in-progress: true jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - uses: pre-commit/action@v3.0.0 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..a6055b1d --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,111 @@ +--- +# yamllint disable rule:line-length +name: run bamboo + +# yamllint disable-line rule:truthy +on: + workflow_call: + workflow_dispatch: + +concurrency: + group: test-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} + cancel-in-progress: true + +jobs: + run-bamboo: + strategy: + fail-fast: false + matrix: + os: + - macos-14 + - ubuntu-24.04 + build-type: + - sstmainline_config + - sstmainline_coreonly_config + - sstmainline_config_no_mpi + - sstmainline_coreonly_config_no_mpi + - sstmainline_config_make_dist_test + - sstmainline_config_core_make_dist_test + exclude: + - os: macos-14 + build-type: sstmainline_config + - os: macos-14 + build-type: sstmainline_coreonly_config + - os: macos-14 + build-type: sstmainline_config_make_dist_test + - os: macos-14 + build-type: sstmainline_config_core_make_dist_test + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: install system-level dependencies (macOS) + run: | + brew install \ + autoconf \ + automake \ + coreutils \ + doxygen \ + libtool \ + lmod \ + ncurses \ + pygments + python -m pip install blessings + # echo "source /opt/homebrew/opt/lmod/init/profile" >> ~/.bashrc + if: matrix.os == 'macos-14' + - name: install system-level dependencies (Ubuntu) + run: | + export DEBIAN_FRONTEND=noninteractive + sudo apt-get -y update + sudo apt-get -y install \ + autoconf \ + automake \ + doxygen \ + libopenmpi-dev \ + libtool-bin \ + lmod \ + python3-blessings \ + python3-pygments + # echo "source /usr/share/lmod/lmod/init/bash" >> ~/.bashrc + if: matrix.os == 'ubuntu-24.04' + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + create-symlink: true + key: ${{ matrix.os }}-${{ matrix.build-type }} + verbose: 2 + - name: set env and prepare dir structure + run: | + # Keep this common in order to increase ccache effectiveness + # sst_deps_user_dir="${GITHUB_WORKSPACE}/../${GITHUB_RUN_ID}_${GITHUB_RUN_NUMBER}_${GITHUB_RUN_ATTEMPT}" + sst_deps_user_dir="${GITHUB_WORKSPACE}/../sst-deps-user-dir" + mkdir -p "${sst_deps_user_dir}" + SST_DEPS_USER_DIR="$(realpath "${sst_deps_user_dir}")" + echo "SST_DEPS_USER_DIR=${SST_DEPS_USER_DIR}" >> "${GITHUB_ENV}" + - name: fetch elements dependencies + run: | + pristine="${SST_DEPS_USER_DIR}"/sstDeps/src/pristine + mkdir -p "${pristine}" + cd "${pristine}" + wget https://github.com/umd-memsys/DRAMSim2/archive/refs/tags/v2.2.2.tar.gz + - name: run bamboo + run: | + # needed for ncurses part of interactive sst-info + export TERM=dumb + MAKEFLAGS="-j$(nproc)" + export MAKEFLAGS + if [[ "$(uname)" == "Darwin" ]]; then + export PMIX_MCA_gds=hash + fi + export SST_DEPS_USER_MODE=1 + mkdir -p "${SST_DEPS_USER_DIR}/devel/trunk" + mv "${GITHUB_WORKSPACE}" "${SST_DEPS_USER_DIR}/devel/sqe" + cd "${SST_DEPS_USER_DIR}/devel/trunk" + ../sqe/buildsys/bamboo.sh ${{ matrix.build-type }} none none none none none + ccache -s -vv + - uses: actions/upload-artifact@v4 + with: + name: test-output-${{ matrix.os }}-${{ matrix.build-type }} + path: sst_test_outputs + compression-level: 9 + if: '!cancelled()' +# yamllint enable rule:line-length diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 817cce1b..9137ec50 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,8 +16,9 @@ repos: - id: fix-byte-order-marker - id: mixed-line-ending args: [--fix, "lf"] - - id: no-commit-to-branch - args: [--branch, main, --branch, master, --branch, devel] + # TODO disable until we can skip on merge commits to main/master/devel + # - id: no-commit-to-branch + # args: [--branch, main, --branch, master, --branch, devel] - id: trailing-whitespace exclude: | (?x)( diff --git a/buildsys/bamboo.sh b/buildsys/bamboo.sh index e3632781..ca780122 100755 --- a/buildsys/bamboo.sh +++ b/buildsys/bamboo.sh @@ -445,6 +445,28 @@ getconfig() { externalelementConfigStr="${NOBUILD}" junoConfigStr="${NOBUILD}" ;; + + sstmainline_coreonly_config_no_mpi) + #----------------------------------------------------------------- + # sstmainline_coreonly_config + # This option used for configuring SST with supported stabledevel deps + #----------------------------------------------------------------- + if [[ ${MPIHOME:+isSet} == isSet ]] ; then + echo ' ' ; echo " Test is flawed! MPI module is loaded!" ; echo ' ' + exit 1 + fi + export | egrep SST_DEPS_ + coreMiscEnv="${cc_environment}" + elementsMiscEnv="${cc_environment}" + depsStr="-r none" # Dependencies only needed for elements + setConvenienceVars "$depsStr" + coreConfigStr="$corebaseoptions $coreMiscEnv --disable-mpi" + elementsConfigStr="${NOBUILD}" + macroConfigStr="${NOBUILD}" + externalelementConfigStr="${NOBUILD}" + junoConfigStr="${NOBUILD}" + ;; + sstmainline_config_linux_with_cuda) #----------------------------------------------------------------- # sstmainline_config_linux_with_cuda @@ -675,9 +697,6 @@ getconfig() { ;; *) - #----------------------------------------------------------------- - # Unrecognized Scenario, This is an error in the bamboo code - #----------------------------------------------------------------- echo ' ' ; echo "Unrecognized Scenario, This is an error in the bamboo code" echo " UNRECOGNIZED: ${1}" exit 1 @@ -697,6 +716,31 @@ getconfig() { fi } +set_up_environment_modules() { + # For some reason, .bashrc is not being run prior to + # this script. Kludge initialization of modules. + + echo "Attempt to initialize the modules utility" + + locations=(/etc/profile.modules /etc/profile.d/modules.sh /opt/homebrew/opt/lmod/init/profile /usr/share/lmod/lmod/init/profile) + + for location in "${locations[@]}"; do + if [ -r "${location}" ]; then + # shellcheck disable=SC1090 + source "${location}" + echo "bamboo.sh: loaded ${location}" + fi + done + + echo "Testing modules utility via ModuleEx..." + echo "ModuleEx avail" + ModuleEx avail + if [ $? -ne 0 ] ; then + echo " ModuleEx Failed" + exit 1 + fi +} + #------------------------------------------------------------------------- # Function: linuxSetMPI # Description: @@ -710,28 +754,7 @@ getconfig() { # Return value: linuxSetMPI() { - # For some reason, .bashrc is not being run prior to - # this script. Kludge initialization of modules. - - echo "Attempt to initialize the modules utility. Look for modules init file in 1 of 2 places" - - echo "Location 1: ls -l /etc/profile.modules" - echo "Location 2: ls -l /etc/profile.d/modules.sh" - if [ -r /etc/profile.modules ] ; then - source /etc/profile.modules - echo "bamboo.sh: loaded /etc/profile.modules" - elif [ -r /etc/profile.d/modules.sh ] ; then - source /etc/profile.d/modules.sh - echo "bamboo.sh: loaded /etc/profile.d/modules" - fi - - echo "Testing modules utility via ModuleEx..." - echo "ModuleEx avail" - ModuleEx avail - if [ $? -ne 0 ] ; then - echo " ModuleEx Failed" - exit 1 - fi + set_up_environment_modules # build MPI selector if [[ "$2" =~ openmpi.* ]] @@ -754,10 +777,7 @@ linuxSetMPI() { elif [[ "$3" =~ intel.* ]] then ModuleEx load intel/${4} - if [[ "$3" == *intel-15* ]] ; then - ModuleEx load gcc/gcc-4.8.1 - fi - + echo "LOADED intel/${4} compiler" fi fi @@ -853,28 +873,19 @@ darwinSetMPI() { macosVersion=`echo ${macosVersionFull} | awk -F. '{print $1 "." $2 }'` echo " ******************* macosVersion= $macosVersion " - # macports or hybrid clang/macports PATH="/opt/local/bin:/usr/local/bin:$PATH" export PATH # Point to aclocal per instructions from sourceforge on MacOSX installation export ACLOCAL_FLAGS="-I/opt/local/share/aclocal $ACLOCAL_FLAGS" - echo $ACLOCAL_FLAGS + echo "ACLOCAL_FLAGS=${ACLOCAL_FLAGS}" - # Initialize modules for Jenkins (taken from $HOME/.bashrc on Mac) - if [ -f /etc/profile.modules ] - then - source /etc/profile.modules - echo "bamboo.sh: loaded /etc/profile.modules." - # put any module loads here - echo "bamboo.sh: Loading Modules for MacOSX" - ldModules_MacOS_Clang $compiler $2 # any Xcode + set_up_environment_modules - else - echo "ERROR: unable to locate /etc/profile.modules - cannot load modules" - exit - fi + # Initialize modules for Jenkins (taken from $HOME/.bashrc on Mac) + echo "bamboo.sh: Loading Modules for MacOSX" + ldModules_MacOS_Clang $compiler $2 # any Xcode echo "bamboo.sh: MacOS build." echo "bamboo.sh: MPI = $2" @@ -1772,6 +1783,10 @@ export SST_BUILD_TYPE="" # - If false, use each given branch as-is. export SST_TEST_MERGE=${SST_TEST_MERGE:-true} +# Number of threads and (MPI) ranks to run tests with (not compile with) +export SST_MULTI_THREAD_COUNT=${SST_MULTI_THREAD_COUNT:-1} +export SST_MULTI_RANK_COUNT=${SST_MULTI_RANK_COUNT:-1} + cloneOtherRepos # Load test definitions @@ -1844,7 +1859,7 @@ else echo "bamboo.sh: KERNEL = $kernel" case ${build_type} in - sstmainline_config|sstmainline_coreonly_config|sstmainline_config_no_gem5|sstmainline_config_clang_core_only|sstmainline_config_macosx_no_gem5|sstmainline_config_no_mpi|sstmainline_config_make_dist_test|sstmainline_config_core_make_dist_test|documentation|sstmainline_config_all|sstmainline_config_linux_with_cuda|sstmainline_config_linux_with_cuda_no_mpi|sst-macro_withsstcore_mac|sst-macro_nosstcore_mac|sst-macro_withsstcore_linux|sst-macro_nosstcore_linux|sst_Macro_make_dist) + sstmainline_config|sstmainline_coreonly_config|sstmainline_coreonly_config_no_mpi|sstmainline_config_no_gem5|sstmainline_config_clang_core_only|sstmainline_config_macosx_no_gem5|sstmainline_config_no_mpi|sstmainline_config_make_dist_test|sstmainline_config_core_make_dist_test|documentation|sstmainline_config_all|sstmainline_config_linux_with_cuda|sstmainline_config_linux_with_cuda_no_mpi|sst-macro_withsstcore_mac|sst-macro_nosstcore_mac|sst-macro_withsstcore_linux|sst-macro_nosstcore_linux|sst_Macro_make_dist) # Save Parameters in case they are needed later SST_DIST_MPI=${mpi_type} _UNUSED="none" diff --git a/buildsys/deps/include/depsDefinitions.sh b/buildsys/deps/include/depsDefinitions.sh index 809b7e5d..34f3a924 100755 --- a/buildsys/deps/include/depsDefinitions.sh +++ b/buildsys/deps/include/depsDefinitions.sh @@ -64,6 +64,10 @@ export SST_DEPS_SRC=$SST_BASE/sstDeps/src # be tar.gz files, zipfiles, or other archive formats. This can be # thought of as the "pre-staging" area for SST dependencies. export SST_DEPS_SRC_PRISTINE=${SST_DEPS_SRC}/pristine +if [[ ! -d "${SST_DEPS_SRC_PRISTINE}" ]]; then + echo "SST_DEPS_SRC_PRISTINE doesn't exist: ${SST_DEPS_SRC_PRISTINE}" + exit 1 +fi # Location where pristine source files are unarchived and patched as a # precursor to build and installation. This can be thought of as the