diff --git a/CMakeLists.txt b/CMakeLists.txt index db6280fbd..1d22f3e97 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.21) # Needed for CUDA, MPI, and CTest features +cmake_minimum_required(VERSION 3.25) # Needed for CUDA, MPI, and CTest features project( EXP @@ -121,6 +121,9 @@ if(OpenMP_FOUND) OPTION (USE_OpenMP "Use OpenMP" ON) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") + if(ENABLE_CUDA) + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler='${OpenMP_CXX_FLAGS}'") + endif() endif() # Slurm support if(SLURM_FOUND) diff --git a/expui/CMakeLists.txt b/expui/CMakeLists.txt index 8690817f7..7e55a3667 100644 --- a/expui/CMakeLists.txt +++ b/expui/CMakeLists.txt @@ -16,8 +16,12 @@ set(common_INCLUDE $ if(ENABLE_CUDA) - list(APPEND common_INCLUDE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} ${CUDAToolkit_INCLUDE_DIRS}) - list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) + if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) + list(APPEND common_LINKLIB CUDA::nvtx3) + else () + list(APPEND common_LINKLIB CUDA::nvToolsExt) + endif () endif() if(SLURM_FOUND) diff --git a/exputil/CMakeLists.txt b/exputil/CMakeLists.txt index 87a25087a..9ccb58eca 100644 --- a/exputil/CMakeLists.txt +++ b/exputil/CMakeLists.txt @@ -51,13 +51,18 @@ set(common_INCLUDE_DIRS $ ${DEP_INC} ${EIGEN3_INCLUDE_DIR} ${HDF5_INCLUDE_DIRS} ${FFTW_INCLUDE_DIRS}) -set(common_LINKLIBS ${DEP_LIB} OpenMP::OpenMP_CXX MPI::MPI_CXX +set(common_LINKLIB ${DEP_LIB} OpenMP::OpenMP_CXX MPI::MPI_CXX yaml-cpp ${VTK_LIBRARIES} ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES} ${FFTW_DOUBLE_LIB}) if(ENABLE_CUDA) - list(APPEND common_INCLUDE_DIRS ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} ${CUDAToolkit_INCLUDE_DIRS}) - list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) + if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) + list(APPEND common_LINKLIB CUDA::nvtx3) + set_source_files_properties(${BIORTH_SRC} ${PARTICLE_SRC} PROPERTIES LANGUAGE CUDA) + else () + list(APPEND common_LINKLIB CUDA::nvToolsExt) + endif () endif() if(ENABLE_XDR AND TIRPC_FOUND) @@ -70,7 +75,7 @@ endif() add_library(exputil ${exputil_SOURCES}) set_target_properties(exputil PROPERTIES OUTPUT_NAME exputil) target_include_directories(exputil PUBLIC ${common_INCLUDE_DIRS}) -target_link_libraries(exputil PUBLIC ${common_LINKLIBS}) +target_link_libraries(exputil PUBLIC ${common_LINKLIB}) install(TARGETS exputil DESTINATION lib) diff --git a/pyEXP/CMakeLists.txt b/pyEXP/CMakeLists.txt index d47080e6e..dc8cebf0e 100644 --- a/pyEXP/CMakeLists.txt +++ b/pyEXP/CMakeLists.txt @@ -17,7 +17,12 @@ set(common_INCLUDE $ ${HDF5_INCLUDE_DIRS} ${EIGEN3_INCLUDE_DIR}) if(ENABLE_CUDA) - list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) + if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) + list(APPEND common_LINKLIB CUDA::nvtx3) + else () + list(APPEND common_LINKLIB CUDA::nvToolsExt) + endif () endif() if(SLURM_FOUND) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 04bd285ae..45827e5f9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,12 +1,3 @@ - -set(CUDA_SRC) -if (ENABLE_CUDA) - list(APPEND CUDA_SRC cudaPolarBasis.cu cudaSphericalBasis.cu - cudaCylinder.cu cudaEmpCylSL.cu cudaComponent.cu NVTX.cc - cudaIncpos.cu cudaIncvel.cu cudaMultistep.cu cudaOrient.cu - cudaBiorthCyl.cu cudaCube.cu cudaSlabSL.cu) -endif() - set(exp_SOURCES Basis.cc Bessel.cc Component.cc Cube.cc Cylinder.cc ExternalCollection.cc ExternalForce.cc Orient.cc PotAccel.cc ScatterMFP.cc @@ -19,9 +10,16 @@ set(exp_SOURCES Basis.cc Bessel.cc Component.cc OutMulti.cc OutRelaxation.cc OrbTrace.cc OutDiag.cc OutLog.cc OutVel.cc OutCoef.cc multistep.cc parse.cc SlabSL.cc step.cc tidalField.cc ultra.cc ultrasphere.cc MPL.cc OutFrac.cc OutCalbr.cc - ParticleFerry.cc chkSlurm.c chkTimer.cc GravKernel.cc ${CUDA_SRC} + ParticleFerry.cc chkSlurm.c chkTimer.cc GravKernel.cc CenterFile.cc PolarBasis.cc FlatDisk.cc signals.cc) +if (ENABLE_CUDA) + list(APPEND exp_SOURCES cudaPolarBasis.cu cudaSphericalBasis.cu + cudaCylinder.cu cudaEmpCylSL.cu cudaComponent.cu NVTX.cc + cudaIncpos.cu cudaIncvel.cu cudaMultistep.cu cudaOrient.cu + cudaBiorthCyl.cu cudaCube.cu cudaSlabSL.cu) +endif() + set(common_INCLUDE_DIRS $ $ @@ -39,8 +37,13 @@ if(PNG_FOUND) endif() if(ENABLE_CUDA) - list(APPEND common_INCLUDE_DIRS ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} ${CUDAToolkit_INCLUDE_DIRS}) - list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) + if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) + list(APPEND common_LINKLIB CUDA::nvtx3) + set_source_files_properties(${exp_SOURCES} PROPERTIES LANGUAGE CUDA) + else () + list(APPEND common_LINKLIB CUDA::nvToolsExt) + endif () endif() if(SLURM_FOUND) @@ -64,5 +67,9 @@ add_executable(exp expand.cc) target_include_directories(exp PUBLIC ${common_INCLUDE_DIRS}) target_link_libraries(exp PUBLIC ${common_LINKLIB} EXPlib) +if (ENABLE_CUDA) + set_target_properties(exp PROPERTIES LINKER_LANGUAGE CUDA) +endif () + install(TARGETS EXPlib DESTINATION lib) install(TARGETS exp DESTINATION bin) diff --git a/src/Cylinder.H b/src/Cylinder.H index 8dc55b14b..08e012c58 100644 --- a/src/Cylinder.H +++ b/src/Cylinder.H @@ -14,7 +14,6 @@ #include #include #endif -#include #include @@ -155,15 +154,7 @@ private: a complete set of coefficients for force evaluation at an intermediate time step */ - void compute_multistep_coefficients() - { - if (play_back and not play_cnew) return; - - nvTracerPtr tPtr; - if (cuda_prof) - tPtr = nvTracerPtr(new nvTracer("Cylinder::compute_multistep_coefficients")); - ortho->compute_multistep_coefficients(mfirst[mstep]); - } + void compute_multistep_coefficients(); //! Reset used particle counter and mass registers virtual void multistep_reset(); diff --git a/src/Cylinder.cc b/src/Cylinder.cc index e17fd50e1..e355bcdf9 100644 --- a/src/Cylinder.cc +++ b/src/Cylinder.cc @@ -12,6 +12,7 @@ #include #include #include +#include Timer timer_debug; @@ -1659,6 +1660,16 @@ void Cylinder::multistep_update(int from, int to, Component* c, int i, int id) #endif } +void Cylinder::compute_multistep_coefficients() +{ + if (play_back and not play_cnew) return; + + nvTracerPtr tPtr; + if (cuda_prof) + tPtr = nvTracerPtr(new nvTracer("Cylinder::compute_multistep_coefficients")); + ortho->compute_multistep_coefficients(mfirst[mstep]); +} + void Cylinder::multistep_reset() { diff --git a/src/ExternalForce.H b/src/ExternalForce.H index 8d8d0f67c..f445852cb 100644 --- a/src/ExternalForce.H +++ b/src/ExternalForce.H @@ -53,6 +53,7 @@ public: //! Finish and clean-up (caching data necessary for restart) virtual void finish() {} + // #if HAVE_LIBCUDA==1 #if HAVE_LIBCUDA==1 //! Copy particles from device for non-cuda forces void getParticlesCuda(Component *c); diff --git a/src/NVTX.H b/src/NVTX.H index dab8d3bcf..44947c197 100644 --- a/src/NVTX.H +++ b/src/NVTX.H @@ -7,7 +7,11 @@ #if HAVE_LIBCUDA==1 +#if __CUDACC_VER_MAJOR__ < 12 #include +#else +#include +#endif class nvTracer { private: diff --git a/src/NVTX.cc b/src/NVTX.cc index 8c4293eb6..60aabeca5 100644 --- a/src/NVTX.cc +++ b/src/NVTX.cc @@ -1,4 +1,4 @@ -#include +#include "NVTX.H" #if HAVE_LIBCUDA==1 diff --git a/src/Orient.H b/src/Orient.H index 13e6ac55a..b2bb1fa91 100644 --- a/src/Orient.H +++ b/src/Orient.H @@ -1,5 +1,3 @@ -// This may look like C code, but it is really -*- C++ -*- - #ifndef _Orient_H #define _Orient_H diff --git a/src/cudaCylinder.cu b/src/cudaCylinder.cu index 8c978ea6b..fff4d19d7 100644 --- a/src/cudaCylinder.cu +++ b/src/cudaCylinder.cu @@ -3,6 +3,7 @@ #include #include #include +#include #include "expand.H" // Define for debugging diff --git a/src/cudaPolarBasis.cu b/src/cudaPolarBasis.cu index 226252aaf..0e51e34ce 100644 --- a/src/cudaPolarBasis.cu +++ b/src/cudaPolarBasis.cu @@ -3,6 +3,7 @@ #include #include #include +#include #include "expand.H" // Define for debugging diff --git a/src/global.H b/src/global.H index 658127f37..0dea91378 100644 --- a/src/global.H +++ b/src/global.H @@ -345,9 +345,11 @@ extern bool cuda_prof; #include "Species.H" #if HAVE_LIBCUDA==1 +extern int cudaGlobalDevice; +#if defined (__NVCC__) #include extern thrust::device_vector cuDstepL, cuDstepN; -extern int cudaGlobalDevice; +#endif #endif #endif diff --git a/src/user/CMakeLists.txt b/src/user/CMakeLists.txt index 3f2a5ff8a..e340cce7e 100644 --- a/src/user/CMakeLists.txt +++ b/src/user/CMakeLists.txt @@ -9,12 +9,19 @@ set (common_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/..) -set (common_LINKLIBS ${DEP_LIB} OpenMP::OpenMP_CXX MPI::MPI_CXX +set (common_LINKLIB ${DEP_LIB} OpenMP::OpenMP_CXX MPI::MPI_CXX exputil EXPlib yaml-cpp) if(ENABLE_CUDA) - list(APPEND common_INCLUDE_DIRS ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} ${CUDAToolkit_INCLUDE_DIRS}) - list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) + if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) + list(APPEND common_LINKLIB CUDA::nvtx3) + else () + list(APPEND common_LINKLIB CUDA::nvToolsExt) + endif () + # set_source_files_properties(UserBar.cc UserDisk.cc UserHalo.cc + # UserLogPot.cc UserMNdisk.cc UserMW.cc UserTest.cc UserTestCuda.cc + # PROPERTIES LANGUAGE CUDA) endif() set(user_SRC UserTest.cc) @@ -33,7 +40,7 @@ foreach(mlib ${USER_MODULES}) add_library(${mlib} ${${mlib}_SRC}) set_target_properties(${mlib} PROPERTIES OUTPUT_NAME ${mlib}) target_include_directories(${mlib} PUBLIC ${common_INCLUDE_DIRS}) - target_link_libraries(${mlib} PUBLIC ${common_LINKLIBS}) + target_link_libraries(${mlib} PUBLIC ${common_LINKLIB}) install(TARGETS ${mlib} DESTINATION lib/user) endforeach() diff --git a/utils/Analysis/CMakeLists.txt b/utils/Analysis/CMakeLists.txt index d06dbe69c..8bc7e3a08 100644 --- a/utils/Analysis/CMakeLists.txt +++ b/utils/Analysis/CMakeLists.txt @@ -13,7 +13,12 @@ if(PNG_FOUND) endif() if(ENABLE_CUDA) - list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) + if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) + list(APPEND common_LINKLIB CUDA::nvtx3) + else () + list(APPEND common_LINKLIB CUDA::nvToolsExt) + endif () endif() if(SLURM_FOUND) diff --git a/utils/ICs/CMakeLists.txt b/utils/ICs/CMakeLists.txt index 06cbb8b18..0c89b9942 100644 --- a/utils/ICs/CMakeLists.txt +++ b/utils/ICs/CMakeLists.txt @@ -9,7 +9,12 @@ set(common_LINKLIB OpenMP::OpenMP_CXX MPI::MPI_CXX yaml-cpp exputil ${HDF5_CXX_LIBRARIES}) if(ENABLE_CUDA) - list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) + if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) + list(APPEND common_LINKLIB CUDA::nvtx3) + else () + list(APPEND common_LINKLIB CUDA::nvToolsExt) + endif () endif() if(ENABLE_XDR AND TIRPC_FOUND) diff --git a/utils/ICs/initial.cc b/utils/ICs/initial.cc index d5d949c55..354733a87 100644 --- a/utils/ICs/initial.cc +++ b/utils/ICs/initial.cc @@ -360,7 +360,7 @@ main(int ac, char **av) double RMIN, RCYLMIN, RCYLMAX, SCSPH, RSPHSL, DMFAC, RFACTOR, SHFAC; double X0, Y0, Z0, U0, V0, W0; int RNUM, PNUM, TNUM, VFLAG, DFLAG; - bool expcond, LOGR, CHEBY, SELECT, DUMPCOEF; + bool expcond, LOGR, CHEBY, DUMPCOEF; int CMAPR, CMAPZ, NCHEB, TCHEB, CMTYPE, NDR, NDZ, NHR, NHT, NDP; int LMAX, NMAXH, NMAXD, MMAX, NUMX, NUMY, NOUT, NMAXLIM, NODD, DF; int DIVERGE, DIVERGE2, SEED, itmax; @@ -368,7 +368,7 @@ main(int ac, char **av) double PPower, R_DF, DR_DF; double Hratio, scale_height, scale_length, scale_lenfkN; double disk_mass, gas_mass, gscal_length, ToomreQ, Temp, Tmin; - bool const_height, images, multi, SVD, basis, zeropos, zerovel; + bool const_height, images, multi, basis, zeropos, zerovel; bool report, ignore, evolved, diskmodel; int nhalo, ndisk, ngas, ngparam; std::string hbods, dbods, gbods, suffix, centerfile, halofile1, halofile2; @@ -870,7 +870,6 @@ main(int ac, char **av) EmpCylSL::CMAPZ = CMAPZ; EmpCylSL::VFLAG = VFLAG; EmpCylSL::logarithmic = LOGR; - EmpCylSL::PCAVAR = SELECT; // Create expansion only if needed . . . std::shared_ptr expandd; diff --git a/utils/MSSA/CMakeLists.txt b/utils/MSSA/CMakeLists.txt index c4db00080..f2a2690cc 100644 --- a/utils/MSSA/CMakeLists.txt +++ b/utils/MSSA/CMakeLists.txt @@ -17,7 +17,12 @@ set(common_INCLUDE $ ${HDF5_INCLUDE_DIRS}) if(ENABLE_CUDA) - list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) + if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) + list(APPEND common_LINKLIB CUDA::nvtx3) + else () + list(APPEND common_LINKLIB CUDA::nvToolsExt) + endif () endif() if(SLURM_FOUND) diff --git a/utils/PhaseSpace/CMakeLists.txt b/utils/PhaseSpace/CMakeLists.txt index 71d1c2141..63ced2c1f 100644 --- a/utils/PhaseSpace/CMakeLists.txt +++ b/utils/PhaseSpace/CMakeLists.txt @@ -21,7 +21,12 @@ if(PNG_FOUND) endif() if(ENABLE_CUDA) - list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) + if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) + list(APPEND common_LINKLIB CUDA::nvtx3) + else () + list(APPEND common_LINKLIB CUDA::nvToolsExt) + endif () endif() if(ENABLE_XDR AND TIRPC_FOUND) diff --git a/utils/SL/CMakeLists.txt b/utils/SL/CMakeLists.txt index 55fe30615..59afb08fa 100644 --- a/utils/SL/CMakeLists.txt +++ b/utils/SL/CMakeLists.txt @@ -6,7 +6,12 @@ set(common_LINKLIB OpenMP::OpenMP_CXX MPI::MPI_CXX yaml-cpp exputil ${VTK_LIBRARIES}) if(ENABLE_CUDA) - list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) + if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) + list(APPEND common_LINKLIB CUDA::nvtx3) + else () + list(APPEND common_LINKLIB CUDA::nvToolsExt) + endif () endif() if(ENABLE_XDR AND TIRPC_FOUND) diff --git a/utils/Test/CMakeLists.txt b/utils/Test/CMakeLists.txt index e680d1723..91a38c6ac 100644 --- a/utils/Test/CMakeLists.txt +++ b/utils/Test/CMakeLists.txt @@ -17,7 +17,12 @@ set(common_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/..) if(ENABLE_CUDA) - list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt ) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) + if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) + list(APPEND common_LINKLIB CUDA::nvtx3) + else () + list(APPEND common_LINKLIB CUDA::nvToolsExt) + endif () endif() if(ENABLE_XDR AND TIRPC_FOUND)