Manage testsuite with CTest

* use a test driver to run test cases in parallel
* configure test cases with labels and arguments
* only show test case output on test failure
* allow skipped test cases (error code 5)

.github/workflows/samples.yml

@@ -31,5 +31,5 @@ jobs:
         run: |
           module restore pymbe
           source venv/bin/activate
-          make functional_tests
+          make functional_tests -j $(nproc)
         shell: bash

.github/workflows/testsuite.yml

@@ -37,7 +37,7 @@ jobs:
           module restore pymbe
           source venv/bin/activate
           make pylint
-          make unit_tests COVERAGE=1
+          make unit_tests -j $(nproc) COVERAGE=1
           make docs
           make coverage_xml
         shell: bash

.gitignore

@@ -6,3 +6,4 @@ frames
 __pycache__
 traj*.vtf
 *_system.png
+testsuite/Testing/

Makefile

@@ -29,52 +29,30 @@ COVERAGE_HTML = coverage
 
 # Python executable or launcher, possibly with command line arguments
 PYTHON = python3
-ifeq ($(COVERAGE),1)
-PYTHON := ${PYTHON} -m coverage run --parallel-mode --source=$(CURDIR)
-endif
+
+# number of threads
+THREADS = $(shell echo $(MAKEFLAGS) | grep -oP "\\-j *\\d+")
 
 docs:
 	mkdir -p ./documentation
 	PDOC_ALLOW_EXEC=0 ${PYTHON} -m pdoc ./pyMBE.py -o ./documentation --docformat google
 
 unit_tests:
-	${PYTHON} testsuite/serialization_test.py
-	${PYTHON} testsuite/lj_tests.py
-	${PYTHON} testsuite/set_particle_acidity_test.py
-	${PYTHON} testsuite/bond_tests.py
-	${PYTHON} testsuite/generate_perpendicular_vectors_test.py
-	${PYTHON} testsuite/define_and_create_molecules_unit_tests.py
-	${PYTHON} testsuite/create_molecule_position_test.py
-	${PYTHON} testsuite/seed_test.py
-	${PYTHON} testsuite/read-write-df_test.py
-	${PYTHON} testsuite/parameter_test.py
-	${PYTHON} testsuite/henderson_hasselbalch_tests.py
-	${PYTHON} testsuite/calculate_net_charge_unit_test.py
-	${PYTHON} testsuite/setup_salt_ions_unit_tests.py
-	${PYTHON} testsuite/globular_protein_unit_tests.py
-	${PYTHON} testsuite/analysis_tests.py
-	${PYTHON} testsuite/charge_number_map_tests.py
-	${PYTHON} testsuite/generate_coordinates_tests.py
-	${PYTHON} testsuite/reaction_methods_unit_tests.py
-	${PYTHON} testsuite/determine_reservoir_concentrations_unit_test.py
+	COVERAGE=$(COVERAGE) ctest --output-on-failure $(THREADS) --test-dir testsuite -LE long --timeout 300
 
 functional_tests:
-	${PYTHON} testsuite/cph_ideal_tests.py
-	${PYTHON} testsuite/grxmc_ideal_tests.py
-	${PYTHON} testsuite/peptide_tests.py
-	${PYTHON} testsuite/gcmc_tests.py
-	${PYTHON} testsuite/weak_polyelectrolyte_dialysis_test.py
-	${PYTHON} testsuite/globular_protein_tests.py
+	COVERAGE=$(COVERAGE) ctest --output-on-failure $(THREADS) --test-dir testsuite -L long
 
-tests: unit_tests functional_tests
+tests:
+	COVERAGE=$(COVERAGE) ctest --output-on-failure $(THREADS) --test-dir testsuite
 
 coverage_xml:
-	${PYTHON} -m coverage combine .
+	${PYTHON} -m coverage combine testsuite
 	${PYTHON} -m coverage report
 	${PYTHON} -m coverage xml
 
 coverage_html:
-	${PYTHON} -m coverage combine .
+	${PYTHON} -m coverage combine testsuite
 	${PYTHON} -m coverage html --directory="${COVERAGE_HTML}"
 
 sample:

README.md

@@ -20,6 +20,7 @@ pyMBE provides tools to facilitate building up molecules with complex architectu
 - [Numpy](https://numpy.org/) >=1.23
 - [SciPy](https://scipy.org/) 
 - [pdoc](https://pdoc.dev/) (for building the docs)
+- [CMake](https://cmake.org/) (for running the testsuite)
 
 ## Contents
 
@@ -153,12 +154,18 @@ To make sure your code is valid, please run the testsuite before submitting your
 
 ```sh
 source pymbe/bin/activate
-make tests
+make tests -j4
 deactivate
 ```
 
+Here, `-j4` instructs CTest to run the test cases in parallel using 4 CPU cores.
+This number can be adjusted depending on your hardware specifications.
+You can use `make unit_tests -j4` to run the subset of fast tests, but keep in mind those
+won't be able to detect more serious bugs that only manifest themselves in long simulations.
+You can also run individual test cases directly, for example with `python3 testsuite/parameter_test.py`.
+
 When contributing new features, consider adding a unit test in the `testsuite/`
-folder and a corresponding line in the `testsuite` target of the Makefile.
+folder and a corresponding line in the `testsuite/CTestTestfile.cmake` file.
 
 Every contribution is automatically tested in CI using EESSI (https://www.eessi.io)
 and the [EESSI GitHub Action](https://github.com/marketplace/actions/eessi).

requirements.txt

@@ -6,3 +6,4 @@ biopandas==0.5.1.dev0
 scipy>=1.8.0
 matplotlib>=3.5.1
 tqdm>=4.57.0
+cmake>=3.22.1

testsuite/CTestTestfile.cmake

@@ -0,0 +1,68 @@
+#
+# Copyright (C) 2024 pyMBE-dev team
+#
+# This file is part of pyMBE.
+#
+# pyMBE is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyMBE is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+cmake_minimum_required(VERSION 3.22.1)
+find_package(Python 3.8 REQUIRED COMPONENTS Interpreter NumPy)
+
+file(REAL_PATH "CTestTestfile.cmake" CMAKE_CURRENT_SOURCE_FILE)
+cmake_path(GET CMAKE_CURRENT_SOURCE_FILE PARENT_PATH CMAKE_CURRENT_SOURCE_DIR)
+cmake_path(GET CMAKE_CURRENT_SOURCE_DIR PARENT_PATH CMAKE_SOURCE_DIR)
+
+function(pymbe_add_test)
+  cmake_parse_arguments(TEST "" "PATH;NAME" "LABELS;ARGUMENTS" ${ARGN})
+  if(NOT DEFINED TEST_NAME)
+    cmake_path(GET TEST_PATH STEM TEST_NAME)
+    set(TEST_NAME ${TEST_NAME} PARENT_SCOPE)
+  endif()
+  if(DEFINED ENV{COVERAGE} AND "$ENV{COVERAGE}" STREQUAL "1")
+    list(APPEND PYTHON_ARGUMENTS "-m" "coverage" "run" "--parallel-mode" "--source=${CMAKE_SOURCE_DIR}")
+  endif()
+  add_test(${TEST_NAME} "${Python_EXECUTABLE}" ${PYTHON_ARGUMENTS} "${TEST_PATH}" ${TEST_ARGUMENTS})
+  set_tests_properties(${TEST_NAME} PROPERTIES SKIP_RETURN_CODE 5)
+  set_tests_properties(${TEST_NAME} PROPERTIES LABELS ${TEST_LABELS})
+endfunction()
+
+# unit tests
+pymbe_add_test(PATH serialization_test.py)
+pymbe_add_test(PATH lj_tests.py)
+pymbe_add_test(PATH set_particle_acidity_test.py)
+pymbe_add_test(PATH bond_tests.py)
+pymbe_add_test(PATH generate_perpendicular_vectors_test.py)
+pymbe_add_test(PATH define_and_create_molecules_unit_tests.py)
+pymbe_add_test(PATH create_molecule_position_test.py)
+pymbe_add_test(PATH seed_test.py)
+pymbe_add_test(PATH read-write-df_test.py)
+pymbe_add_test(PATH parameter_test.py)
+pymbe_add_test(PATH henderson_hasselbalch_tests.py)
+pymbe_add_test(PATH calculate_net_charge_unit_test.py)
+pymbe_add_test(PATH setup_salt_ions_unit_tests.py)
+pymbe_add_test(PATH globular_protein_unit_tests.py)
+pymbe_add_test(PATH analysis_tests.py)
+pymbe_add_test(PATH charge_number_map_tests.py)
+pymbe_add_test(PATH generate_coordinates_tests.py)
+pymbe_add_test(PATH reaction_methods_unit_tests.py)
+pymbe_add_test(PATH determine_reservoir_concentrations_unit_test.py)
+
+# functional tests, e.g. long simulations and ensemble averages
+pymbe_add_test(PATH globular_protein_tests.py LABELS long)
+pymbe_add_test(PATH peptide_tests.py LABELS long)
+pymbe_add_test(PATH weak_polyelectrolyte_dialysis_test.py LABELS long)
+pymbe_add_test(PATH cph_ideal_tests.py LABELS long)
+pymbe_add_test(PATH grxmc_ideal_tests.py LABELS long)
+pymbe_add_test(PATH gcmc_tests.py LABELS long)