From 698f75cb47e8c641b8c4cde1eee15ff18b217b16 Mon Sep 17 00:00:00 2001 From: Evan Weinberg Date: Mon, 17 Jul 2023 11:37:09 -0700 Subject: [PATCH 1/7] Hotfix to broken staggered Mat host verify introduced in 931680a --- tests/host_reference/staggered_dslash_reference.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/host_reference/staggered_dslash_reference.cpp b/tests/host_reference/staggered_dslash_reference.cpp index e95532590f..86ecd17464 100644 --- a/tests/host_reference/staggered_dslash_reference.cpp +++ b/tests/host_reference/staggered_dslash_reference.cpp @@ -116,10 +116,10 @@ void staggeredDslashReference(sFloat *res, gFloat **fatlink, gFloat **longlink, sub(&res[offset], &res[offset], gaugedSpinor, stag_spinor_site_size); } } + } // forward/backward in all four directions - if (daggerBit) negx(&res[offset], stag_spinor_site_size); - } // 4-d volume - } // right-hand-side + if (daggerBit) negx(&res[offset], stag_spinor_site_size); + } // 4-d volume } void staggeredDslash(ColorSpinorField &out, void **fatlink, void **longlink, void **ghost_fatlink, From 0e7bef445757933bcd7ccf4688af96c153feaca1 Mon Sep 17 00:00:00 2001 From: Evan Weinberg Date: Mon, 17 Jul 2023 11:51:44 -0700 Subject: [PATCH 2/7] Added staggered/asqtad Mat ctest --- tests/CMakeLists.txt | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9ca912f8b3..5f4c9808ae 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -801,14 +801,24 @@ endif() if(QUDA_DIRAC_STAGGERED) set(DIRAC_NAME staggered) - add_test(NAME dslash_${DIRAC_NAME}_policy${pol2} + add_test(NAME dslash_${DIRAC_NAME}_matpc_policy${pol2} COMMAND ${QUDA_CTEST_LAUNCH} $ ${MPIEXEC_POSTFLAGS} --dslash-type ${DIRAC_NAME} --test MatPC --dim 2 4 6 8 - --gtest_output=xml:dslash_${DIRAC_NAME}_test_pol${pol2}.xml) + --gtest_output=xml:dslash_${DIRAC_NAME}_matpc_test_pol${pol2}.xml) if(polenv) - set_tests_properties(dslash_${DIRAC_NAME}_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol2}) + set_tests_properties(dslash_${DIRAC_NAME}_matpc_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol2}) + endif() + + add_test(NAME dslash_${DIRAC_NAME}_mat_policy${pol2} + COMMAND ${QUDA_CTEST_LAUNCH} $ ${MPIEXEC_POSTFLAGS} + --dslash-type ${DIRAC_NAME} + --test Mat + --dim 2 4 6 8 + --gtest_output=xml:dslash_${DIRAC_NAME}_mat_test_pol${pol2}.xml) + if(polenv) + set_tests_properties(dslash_${DIRAC_NAME}_mat_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol2}) endif() add_test(NAME benchmark_dslash_${DIRAC_NAME}_policy${pol2} @@ -824,15 +834,26 @@ endif() endif() set(DIRAC_NAME asqtad) - add_test(NAME dslash_${DIRAC_NAME}_policy${pol2} + add_test(NAME dslash_${DIRAC_NAME}_matpc_policy${pol2} COMMAND ${QUDA_CTEST_LAUNCH} $ ${MPIEXEC_POSTFLAGS} --dslash-type ${DIRAC_NAME} --all-partitions 1 --test MatPC --dim 6 8 10 12 - --gtest_output=xml:dslash_${DIRAC_NAME}_test_pol${pol2}.xml) + --gtest_output=xml:dslash_${DIRAC_NAME}_matpc_test_pol${pol2}.xml) + if(polenv) + set_tests_properties(dslash_${DIRAC_NAME}_matpc_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol2}) + endif() + + add_test(NAME dslash_${DIRAC_NAME}_mat_policy${pol2} + COMMAND ${QUDA_CTEST_LAUNCH} $ ${MPIEXEC_POSTFLAGS} + --dslash-type ${DIRAC_NAME} + --all-partitions 1 + --test Mat + --dim 6 8 10 12 + --gtest_output=xml:dslash_${DIRAC_NAME}_mat_test_pol${pol2}.xml) if(polenv) - set_tests_properties(dslash_${DIRAC_NAME}_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol2}) + set_tests_properties(dslash_${DIRAC_NAME}_mat_policy${pol2} PROPERTIES ENVIRONMENT QUDA_ENABLE_DSLASH_POLICY=${pol2}) endif() add_test(NAME benchmark_dslash_${DIRAC_NAME}_policy${pol2} From 155d48decc93b90bca8959a63129c0320170ac30 Mon Sep 17 00:00:00 2001 From: Jiqun Tu Date: Thu, 27 Jul 2023 12:09:30 -0400 Subject: [PATCH 3/7] Use size_t for failure count. --- lib/color_spinor_util.in.cu | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/color_spinor_util.in.cu b/lib/color_spinor_util.in.cu index 3681438c9f..f2305dfe81 100644 --- a/lib/color_spinor_util.in.cu +++ b/lib/color_spinor_util.in.cu @@ -191,7 +191,7 @@ namespace quda { int compareSpinor(const U &u, const V &v, const int tol) { int fail_check = 16*tol; - std::vector fail(fail_check); + std::vector fail(fail_check); for (int f=0; f Date: Thu, 27 Jul 2023 13:18:40 -0400 Subject: [PATCH 4/7] Clang-format. --- lib/color_spinor_util.in.cu | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/color_spinor_util.in.cu b/lib/color_spinor_util.in.cu index f2305dfe81..65b2ae1d55 100644 --- a/lib/color_spinor_util.in.cu +++ b/lib/color_spinor_util.in.cu @@ -226,7 +226,7 @@ namespace quda { for (int i=0; i Date: Fri, 28 Jul 2023 10:49:29 -0400 Subject: [PATCH 5/7] Reduce among the ranks for size_t. --- include/communicator_quda.h | 2 ++ lib/color_spinor_util.in.cu | 2 +- lib/communicator_mpi.cpp | 10 ++++++++++ lib/communicator_qmp.cpp | 8 ++++++++ lib/communicator_single.cpp | 2 ++ lib/communicator_stack.cpp | 4 ++++ 6 files changed, 27 insertions(+), 1 deletion(-) diff --git a/include/communicator_quda.h b/include/communicator_quda.h index 19bbf6e42c..0dc22fdfab 100644 --- a/include/communicator_quda.h +++ b/include/communicator_quda.h @@ -741,6 +741,8 @@ namespace quda void comm_allreduce_sum_array(double *data, size_t size); + void comm_allreduce_sum(size_t &a); + void comm_allreduce_max_array(double *data, size_t size); void comm_allreduce_max_array(deviation_t *data, size_t size); diff --git a/lib/color_spinor_util.in.cu b/lib/color_spinor_util.in.cu index 65b2ae1d55..ffe18033f0 100644 --- a/lib/color_spinor_util.in.cu +++ b/lib/color_spinor_util.in.cu @@ -224,7 +224,7 @@ namespace quda { // reduce over all processes for (int i=0; i *data, size_t size) { size_t n = comm_size(); diff --git a/lib/communicator_qmp.cpp b/lib/communicator_qmp.cpp index 174c95fa9d..1a4a2a7878 100644 --- a/lib/communicator_qmp.cpp +++ b/lib/communicator_qmp.cpp @@ -318,6 +318,14 @@ void Communicator::comm_allreduce_sum_array(double *data, size_t size) } } +void Communicator::comm_allreduce_sum(size_t &a) +{ + if (sizeof(size_t) != sizeof(uint64_t)) { + errorQuda("sizeof(size_t) != sizeof(uint64_t): %lu != %lu\n", sizeof(size_t), sizeof(uint64_t)); + } + QMP_CHECK(QMP_comm_sum_uint64_t(QMP_COMM_HANDLE, reinterpret_cast(&a))); +} + void Communicator::comm_allreduce_max_array(deviation_t *data, size_t size) { size_t n = comm_size(); diff --git a/lib/communicator_single.cpp b/lib/communicator_single.cpp index e6757a1265..ed4f48df06 100644 --- a/lib/communicator_single.cpp +++ b/lib/communicator_single.cpp @@ -83,6 +83,8 @@ namespace quda void Communicator::comm_allreduce_sum_array(double *, size_t) { } + void Communicator::comm_allreduce_sum(size_t &a) { } + void Communicator::comm_allreduce_max_array(deviation_t *, size_t) { } void Communicator::comm_allreduce_max_array(double *, size_t) { } diff --git a/lib/communicator_stack.cpp b/lib/communicator_stack.cpp index 9d6f1eb7e3..82bdced142 100644 --- a/lib/communicator_stack.cpp +++ b/lib/communicator_stack.cpp @@ -216,6 +216,10 @@ namespace quda template <> void comm_allreduce_sum(double &a) { comm_allreduce_sum_array(&a, 1); } + template <> void comm_allreduce_sum(size_t &a) { + get_current_communicator().comm_allreduce_sum(a); + } + void comm_allreduce_max_array(double *data, size_t size) { get_current_communicator().comm_allreduce_max_array(data, size); From acb1a89f4d8ffed8140ce66125d950e2f4d95533 Mon Sep 17 00:00:00 2001 From: Jiqun Tu Date: Fri, 28 Jul 2023 14:00:10 -0400 Subject: [PATCH 6/7] Fix compiler warning. --- lib/color_spinor_util.in.cu | 4 ++-- lib/communicator_stack.cpp | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/color_spinor_util.in.cu b/lib/color_spinor_util.in.cu index ffe18033f0..c0baeb0ec5 100644 --- a/lib/color_spinor_util.in.cu +++ b/lib/color_spinor_util.in.cu @@ -224,9 +224,9 @@ namespace quda { // reduce over all processes for (int i=0; i void comm_allreduce_sum(double &a) { comm_allreduce_sum_array(&a, 1); } - template <> void comm_allreduce_sum(size_t &a) { - get_current_communicator().comm_allreduce_sum(a); - } + template <> void comm_allreduce_sum(size_t &a) { get_current_communicator().comm_allreduce_sum(a); } void comm_allreduce_max_array(double *data, size_t size) { From eccf54b45fc173906cbd04bec6943b7c9d304a1c Mon Sep 17 00:00:00 2001 From: Jiqun Tu Date: Fri, 28 Jul 2023 17:28:39 -0400 Subject: [PATCH 7/7] Fix compiler warning. --- lib/communicator_single.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/communicator_single.cpp b/lib/communicator_single.cpp index ed4f48df06..978fbcc069 100644 --- a/lib/communicator_single.cpp +++ b/lib/communicator_single.cpp @@ -83,7 +83,7 @@ namespace quda void Communicator::comm_allreduce_sum_array(double *, size_t) { } - void Communicator::comm_allreduce_sum(size_t &a) { } + void Communicator::comm_allreduce_sum(size_t &) { } void Communicator::comm_allreduce_max_array(deviation_t *, size_t) { }