From 9e536b4f6d2633644b3fb2ea88d142cf67b1fe6b Mon Sep 17 00:00:00 2001 From: Elliot Ronaghan Date: Thu, 11 Mar 2021 10:04:32 -0500 Subject: [PATCH 1/2] Copy cray-cs perf jobs in prep for adding cray-cs-hdr jobs Signed-off-by: Elliot Ronaghan --- util/cron/common-perf-cray-cs-hdr.bash | 13 +++++++++ ...test-perf.cray-cs-hdr.gasnet-ibv.fast.bash | 27 +++++++++++++++++++ ...est-perf.cray-cs-hdr.gasnet-ibv.large.bash | 22 +++++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 util/cron/common-perf-cray-cs-hdr.bash create mode 100755 util/cron/test-perf.cray-cs-hdr.gasnet-ibv.fast.bash create mode 100755 util/cron/test-perf.cray-cs-hdr.gasnet-ibv.large.bash diff --git a/util/cron/common-perf-cray-cs-hdr.bash b/util/cron/common-perf-cray-cs-hdr.bash new file mode 100644 index 000000000000..333884c00bd6 --- /dev/null +++ b/util/cron/common-perf-cray-cs-hdr.bash @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +# Configure settings for Cray CS performance testing. + +export CHPL_LAUNCHER_PARTITION=bdw18 +export CHPL_TARGET_CPU=broadwell + +# the lengths we go to, to avoid line wrap ... +pcca=(-performance-configs gn-ibv-large:v,gn-ibv-fast:v,gn-mpi,ofi:v \ + -performance \ + -perflabel ml- \ + -startdate 07/01/19) +perf_cray_cs_args=${pcca[*]} diff --git a/util/cron/test-perf.cray-cs-hdr.gasnet-ibv.fast.bash b/util/cron/test-perf.cray-cs-hdr.gasnet-ibv.fast.bash new file mode 100755 index 000000000000..5edf29aeffbe --- /dev/null +++ b/util/cron/test-perf.cray-cs-hdr.gasnet-ibv.fast.bash @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# +# Run performance tests on a cray-cs + +CWD=$(cd $(dirname $0) ; pwd) + +export CHPL_TEST_PERF_CONFIG_NAME='16-node-cs' + +source $CWD/common-perf.bash +export CHPL_TEST_PERF_DIR=/cray/css/users/chapelu/NightlyPerformance/cray-cs/16-node-cs + +export CHPL_NIGHTLY_TEST_CONFIG_NAME="perf.cray-cs.gasnet-ibv.fast" + +source $CWD/common-cray-cs.bash y +source $CWD/common-perf-cray-cs.bash + +# Keep long running tests as a performance issue instead of being reported as a +# correctness timeout -- https://github.com/Cray/chapel-private/issues/443 +export CHPL_TEST_TIMEOUT=1800 + +export CHPL_GASNET_SEGMENT=fast +export GASNET_PHYSMEM_MAX=83G +export CHPL_GASNET_MORE_CFG_OPTIONS=--disable-ibv-odp +nightly_args="${nightly_args} -no-buildcheck" +perf_args="-performance-description gn-ibv-fast -numtrials 1" + +$CWD/nightly -cron ${perf_args} ${perf_cray_cs_args} ${nightly_args} diff --git a/util/cron/test-perf.cray-cs-hdr.gasnet-ibv.large.bash b/util/cron/test-perf.cray-cs-hdr.gasnet-ibv.large.bash new file mode 100755 index 000000000000..35fea3405300 --- /dev/null +++ b/util/cron/test-perf.cray-cs-hdr.gasnet-ibv.large.bash @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# Run performance tests on a cray-cs + +CWD=$(cd $(dirname $0) ; pwd) + +export CHPL_TEST_PERF_CONFIG_NAME='16-node-cs' + +source $CWD/common-perf.bash +export CHPL_TEST_PERF_DIR=/cray/css/users/chapelu/NightlyPerformance/cray-cs/16-node-cs + +export CHPL_NIGHTLY_TEST_CONFIG_NAME="perf.cray-cs.gasnet-ibv.large" + +source $CWD/common-cray-cs.bash y +source $CWD/common-perf-cray-cs.bash + +export GASNET_PHYSMEM_MAX=83G +export GASNET_ODP_VERBOSE=0 +nightly_args="${nightly_args} -no-buildcheck" +perf_args="-performance-description gn-ibv-large -numtrials 3" + +$CWD/nightly -cron ${perf_args} ${perf_cray_cs_args} ${nightly_args} From 212a9f8128b8e537d167a7348d0261a32d94edf6 Mon Sep 17 00:00:00 2001 From: Elliot Ronaghan Date: Thu, 11 Mar 2021 10:19:47 -0500 Subject: [PATCH 2/2] Add some cray-cs perf testing on a system with HDR IB This adds some additional testing to a newer Cray CS. To start I'm only adding gasnet-ibv large and fast configurations. Signed-off-by: Elliot Ronaghan --- util/cron/common-perf-cray-cs-hdr.bash | 8 ++++---- .../test-perf.cray-cs-hdr.gasnet-ibv.fast.bash | 15 ++++++--------- .../test-perf.cray-cs-hdr.gasnet-ibv.large.bash | 13 +++++++------ 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/util/cron/common-perf-cray-cs-hdr.bash b/util/cron/common-perf-cray-cs-hdr.bash index 333884c00bd6..fbb095f46fed 100644 --- a/util/cron/common-perf-cray-cs-hdr.bash +++ b/util/cron/common-perf-cray-cs-hdr.bash @@ -1,12 +1,12 @@ #!/usr/bin/env bash # -# Configure settings for Cray CS performance testing. +# Configure settings for Cray CS HDR performance testing. -export CHPL_LAUNCHER_PARTITION=bdw18 -export CHPL_TARGET_CPU=broadwell +export CHPL_LAUNCHER_PARTITION=clx24 +export CHPL_TARGET_CPU=none # the lengths we go to, to avoid line wrap ... -pcca=(-performance-configs gn-ibv-large:v,gn-ibv-fast:v,gn-mpi,ofi:v \ +pcca=(-performance-configs gn-ibv-large:v,gn-ibv-fast:v \ -performance \ -perflabel ml- \ -startdate 07/01/19) diff --git a/util/cron/test-perf.cray-cs-hdr.gasnet-ibv.fast.bash b/util/cron/test-perf.cray-cs-hdr.gasnet-ibv.fast.bash index 5edf29aeffbe..ea7ed5d6857b 100755 --- a/util/cron/test-perf.cray-cs-hdr.gasnet-ibv.fast.bash +++ b/util/cron/test-perf.cray-cs-hdr.gasnet-ibv.fast.bash @@ -4,22 +4,19 @@ CWD=$(cd $(dirname $0) ; pwd) -export CHPL_TEST_PERF_CONFIG_NAME='16-node-cs' +export CHPL_TEST_PERF_CONFIG_NAME='16-node-cs-hdr' source $CWD/common-perf.bash -export CHPL_TEST_PERF_DIR=/cray/css/users/chapelu/NightlyPerformance/cray-cs/16-node-cs +export CHPL_TEST_PERF_DIR=/cray/css/users/chapelu/NightlyPerformance/cray-cs/16-node-cs-hdr -export CHPL_NIGHTLY_TEST_CONFIG_NAME="perf.cray-cs.gasnet-ibv.fast" +export CHPL_NIGHTLY_TEST_CONFIG_NAME="perf.cray-cs-hdr.gasnet-ibv.fast" source $CWD/common-cray-cs.bash y -source $CWD/common-perf-cray-cs.bash - -# Keep long running tests as a performance issue instead of being reported as a -# correctness timeout -- https://github.com/Cray/chapel-private/issues/443 -export CHPL_TEST_TIMEOUT=1800 +source $CWD/common-perf-cray-cs-hdr.bash export CHPL_GASNET_SEGMENT=fast -export GASNET_PHYSMEM_MAX=83G +export GASNET_PHYSMEM_MAX=124G +export GASNET_IBV_PORTS=mlx5_1 export CHPL_GASNET_MORE_CFG_OPTIONS=--disable-ibv-odp nightly_args="${nightly_args} -no-buildcheck" perf_args="-performance-description gn-ibv-fast -numtrials 1" diff --git a/util/cron/test-perf.cray-cs-hdr.gasnet-ibv.large.bash b/util/cron/test-perf.cray-cs-hdr.gasnet-ibv.large.bash index 35fea3405300..c144fab7301c 100755 --- a/util/cron/test-perf.cray-cs-hdr.gasnet-ibv.large.bash +++ b/util/cron/test-perf.cray-cs-hdr.gasnet-ibv.large.bash @@ -4,19 +4,20 @@ CWD=$(cd $(dirname $0) ; pwd) -export CHPL_TEST_PERF_CONFIG_NAME='16-node-cs' +export CHPL_TEST_PERF_CONFIG_NAME='16-node-cs-hdr' source $CWD/common-perf.bash -export CHPL_TEST_PERF_DIR=/cray/css/users/chapelu/NightlyPerformance/cray-cs/16-node-cs +export CHPL_TEST_PERF_DIR=/cray/css/users/chapelu/NightlyPerformance/cray-cs/16-node-cs-hdr -export CHPL_NIGHTLY_TEST_CONFIG_NAME="perf.cray-cs.gasnet-ibv.large" +export CHPL_NIGHTLY_TEST_CONFIG_NAME="perf.cray-cs-hdr.gasnet-ibv.large" source $CWD/common-cray-cs.bash y -source $CWD/common-perf-cray-cs.bash +source $CWD/common-perf-cray-cs-hdr.bash -export GASNET_PHYSMEM_MAX=83G +export GASNET_PHYSMEM_MAX=124G +export GASNET_IBV_PORTS=mlx5_1 export GASNET_ODP_VERBOSE=0 nightly_args="${nightly_args} -no-buildcheck" -perf_args="-performance-description gn-ibv-large -numtrials 3" +perf_args="-performance-description gn-ibv-large -numtrials 1" $CWD/nightly -cron ${perf_args} ${perf_cray_cs_args} ${nightly_args}