diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d2ef064..7fd515b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,8 +1,8 @@ name: Test -on: -- push +on: [push, pull_request] jobs: check: + if: github.event_name != 'push' || github.ref != 'refs/heads/devel' runs-on: ubuntu-latest strategy: matrix: @@ -10,8 +10,6 @@ jobs: name: Check py${{ matrix.python }} steps: - uses: actions/checkout@v2 - with: - fetch-depth: 0 - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python }} @@ -21,54 +19,84 @@ jobs: with: path: ~/.cache/pre-commit key: pre-commit|${{ env.PYSHA }}|${{ hashFiles('.pre-commit-config.yaml') }} - - run: pip install -U .[dev] - - run: python setup.py sdist bdist_wheel - - run: twine check dist/* + - name: dependencies + run: pip install -U pre-commit - uses: reviewdog/action-setup@v1 - if: github.event_name != 'schedule' + name: comment run: | - pre-commit run -a flake8 | reviewdog -f=pep8 -name=flake8 -tee -reporter=github-check -filter-mode nofilter + if [[ $EVENT == pull_request ]]; then + REPORTER=github-pr-review + else + REPORTER=github-check + fi + pre-commit run -a todo | reviewdog -efm="%f:%l: %m" -name=TODO -tee -reporter=$REPORTER -filter-mode nofilter + pre-commit run -a flake8 | reviewdog -f=pep8 -name=flake8 -tee -reporter=$REPORTER -filter-mode nofilter env: REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EVENT: ${{ github.event_name }} - if: startsWith(matrix.python, '3') run: pre-commit run -a --show-diff-on-failure - - if: startsWith(matrix.python, '2') - run: pre-commit run -a --show-diff-on-failure isort test: - runs-on: [self-hosted, matlab] + if: github.event_name != 'pull_request' || github.head_ref != 'devel' + runs-on: ubuntu-latest + strategy: + matrix: + python: [2.7, 3.6, 3.9] + name: Test py${{ matrix.python }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - run: pip install -U .[dev] + - if: startsWith(matrix.python, '3') + run: pytest --durations-min=1 + - if: startsWith(matrix.python, '2') + run: pytest + - run: codecov + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + matlab: + if: github.event_name != 'pull_request' || github.head_ref != 'devel' + runs-on: [self-hosted, python, matlab] name: Test matlab steps: - uses: actions/checkout@v2 with: fetch-depth: 0 + - name: Run setup-python + run: setup-python -p3.7 - run: pip install -U .[dev] - - run: pytest + - run: pytest --durations-min=1 - run: codecov env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + - name: Post Run setup-python + run: setup-python -p3.7 -Dr + if: ${{ always() }} deploy: - if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') - needs: [check, test] + needs: [check, test, matlab] name: PyPI Deploy runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 with: fetch-depth: 0 - - uses: casperdcl/deploy-pypi@v1 + - uses: actions/setup-python@v2 + - id: dist + uses: casperdcl/deploy-pypi@v2 with: build: true gpg_key: ${{ secrets.GPG_KEY }} password: ${{ secrets.PYPI_TOKEN }} - - id: collect_assets - name: Collect assets - run: | - echo "::set-output name=asset_path::$(ls dist/*.whl)" - echo "::set-output name=asset_name::$(basename dist/*.whl)" - echo "::set-output name=asset_path_sig::$(ls dist/*.whl.asc 2>/dev/null)" - echo "::set-output name=asset_name_sig::$(basename dist/*.whl.asc 2>/dev/null)" - git log --pretty='format:%d%n- %s%n%b---' $(git tag --sort=v:refname | tail -n2 | head -n1)..HEAD > _CHANGES.md - - id: create_release + upload: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') }} + - name: Changelog + run: git log --pretty='format:%d%n- %s%n%b---' $(git tag --sort=v:refname | tail -n2 | head -n1)..HEAD > _CHANGES.md + - if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + id: create_release uses: actions/create-release@v1 env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} @@ -77,13 +105,14 @@ jobs: release_name: spm12 ${{ github.ref }} beta body_path: _CHANGES.md draft: true - - uses: actions/upload-release-asset@v1 + - if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ${{ steps.collect_assets.outputs.asset_path }} - asset_name: ${{ steps.collect_assets.outputs.asset_name }} + asset_path: dist/${{ steps.dist.outputs.whl }} + asset_name: ${{ steps.dist.outputs.whl }} asset_content_type: application/zip - if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') uses: actions/upload-release-asset@v1 @@ -91,6 +120,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ${{ steps.collect_assets.outputs.asset_path_sig }} - asset_name: ${{ steps.collect_assets.outputs.asset_name_sig }} + asset_path: dist/${{ steps.dist.outputs.whl_asc }} + asset_name: ${{ steps.dist.outputs.whl_asc }} asset_content_type: text/plain diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9d687f1..2fcbb14 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,31 +2,43 @@ default_language_version: python: python3 repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 + rev: v3.4.0 hooks: - id: check-added-large-files - id: check-case-conflict - id: check-docstring-first - id: check-executables-have-shebangs - id: check-toml + - id: check-merge-conflict - id: check-yaml + - id: debug-statements - id: end-of-file-fixer - id: mixed-line-ending + - id: sort-simple-yaml - id: trailing-whitespace -- hooks: +- repo: local + hooks: + - id: todo + name: Check TODO + language: pygrep + entry: TODO + types: [text] + exclude: ^(.pre-commit-config.yaml|.github/workflows/test.yml)$ + args: [-i] +- repo: https://gitlab.com/pycqa/flake8 + rev: 3.8.4 + hooks: - id: flake8 additional_dependencies: - flake8-bugbear - flake8-comprehensions - flake8-debugger - flake8-string-format - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.4 -- hooks: - - id: black - repo: https://github.com/psf/black +- repo: https://github.com/psf/black rev: 19.10b0 -- hooks: + hooks: + - id: black +- repo: https://github.com/PyCQA/isort + rev: 5.7.0 + hooks: - id: isort - repo: https://github.com/timothycrosley/isort - rev: 5.6.4 diff --git a/setup.cfg b/setup.cfg index bf22734..66aa44b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,6 @@ classifiers= Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 @@ -48,11 +47,13 @@ install_requires= scipy include_package_data=True packages=find: +python_requires=>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.* [options.extras_require] dev= pre-commit twine wheel + miutil[web]>=0.7.0 pytest pytest-cov pytest-timeout @@ -80,4 +81,7 @@ profile=black known_first_party=spm12,tests [tool:pytest] -addopts=-v --tb=short -rxs -W=error --log-level=debug -n=auto --durations=0 --durations-min=1 --cov=spm12 --cov-report=term-missing --cov-report=xml +timeout=300 +addopts=-v --tb=short -rxs -W=error --log-level=debug -n=auto --durations=0 --cov=spm12 --cov-report=term-missing --cov-report=xml +filterwarnings= + ignore:numpy.ufunc size changed.*:RuntimeWarning diff --git a/spm12/utils.py b/spm12/utils.py index e2ca2bf..da1036d 100644 --- a/spm12/utils.py +++ b/spm12/utils.py @@ -1,11 +1,16 @@ import logging -from functools import lru_cache, wraps +from functools import wraps from os import path from textwrap import dedent from miutil.mlab import get_engine from pkg_resources import resource_filename +try: + from functools import lru_cache +except ImportError: # fix py2.7 + from backports.functools_lru_cache import lru_cache + __all__ = ["get_matlab", "ensure_spm"] PATH_M = resource_filename(__name__, "") log = logging.getLogger(__name__) @@ -30,7 +35,7 @@ def ensure_spm(name=None, cache="~/.spm12", version=12): if path.exists(addpath): eng.addpath(addpath) if not eng.exist("spm_jobman"): - log.warn("MATLAB could not find SPM.") + log.warning("MATLAB could not find SPM.") try: from zipfile import ZipFile diff --git a/tests/__main__.py b/tests/__main__.py new file mode 100644 index 0000000..b4ee5a5 --- /dev/null +++ b/tests/__main__.py @@ -0,0 +1,22 @@ +import logging + +from miutil.fdio import extractall +from miutil.web import urlopen_cached + +from .conftest import HOME + +log = logging.getLogger(__name__) +DATA_URL = ( + "https://zenodo.org/record/3877529/files/amyloidPET_FBP_TP0_extra.zip?download=1" +) + + +def main(): + log.info(f"Downloading {DATA_URL}\nto ${{DATA_ROOT:-~}} ({HOME})") + with urlopen_cached(DATA_URL, HOME) as fd: + extractall(fd, HOME) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..8206804 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,54 @@ +from os import getenv +from textwrap import dedent + +from miutil.fdio import Path +from pytest import fixture, skip + +HOME = Path(getenv("DATA_ROOT", "~")).expanduser() + + +@fixture(scope="session") +def folder_in(): + Ab_PET_mMR_test = HOME / "Ab_PET_mMR_test" + if not Ab_PET_mMR_test.is_dir(): + skip( + dedent( + """\ + Cannot find Ab_PET_mMR_test in ${DATA_ROOT:-~} (%s). + Try running `python -m tests` to download it. + """ + ) + % HOME + ) + return Ab_PET_mMR_test + + +@fixture(scope="session") +def folder_ref(folder_in): + Ab_PET_mMR_ref = folder_in / "testing_reference" / "Ab_PET_mMR_ref" + if not Ab_PET_mMR_ref.is_dir(): + skip( + dedent( + """\ + Cannot find Ab_PET_mMR_ref in + ${DATA_ROOT:-~}/testing_reference (%s/testing_reference). + Try running `python -m tests` to download it. + """ + ) + % HOME + ) + return Ab_PET_mMR_ref + + +@fixture +def PET(folder_ref): + res = folder_ref / "basic" / "17598013_t-3000-3600sec_itr-4_suvr.nii.gz" + assert res.is_file() + return res + + +@fixture +def MRI(folder_in): + res = folder_in / "T1w_N4" / "t1_S00113_17598013_N4bias_cut.nii.gz" + assert res.is_file() + return res diff --git a/tests/test_main.py b/tests/test_main.py index 7bfc8f1..2b68260 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,9 +1,19 @@ from miutil.mlab import matlabroot -from pytest import skip +from pytest import mark, skip + +try: + FileNotFoundError +except NameError: # fix py2.7 + FileNotFoundError = OSError + +pytestmark = mark.filterwarnings("ignore:numpy.ufunc size changed.*:RuntimeWarning") def test_cli(): - if matlabroot("None") == "None": + try: + if matlabroot("None") == "None": + raise FileNotFoundError + except FileNotFoundError: skip("MATLAB not installed") from spm12.cli import main diff --git a/tests/test_regseg.py b/tests/test_regseg.py index e683a60..a3865e8 100644 --- a/tests/test_regseg.py +++ b/tests/test_regseg.py @@ -1,23 +1,11 @@ -from os import getenv, path from textwrap import dedent import numpy as np -import pytest -from miutil import tmpdir from miutil.imio import nii +from pytest import mark from spm12 import regseg -HOME = getenv("DATA_ROOT", path.expanduser("~")) -DATA = path.join(HOME, "Ab_PET_mMR_test") -MRI = path.join(DATA, "T1w_N4", "t1_S00113_17598013_N4bias_cut.nii.gz") -PET = path.join( - DATA, - "testing_reference", - "Ab_PET_mMR_ref", - "basic", - "17598013_t-3000-3600sec_itr-4_suvr.nii.gz", -) MRI2PET = np.array( [ [0.99990508, 0.00800995, 0.01121016, -0.68164088], @@ -26,17 +14,8 @@ [0.0, 0.0, 0.0, 1.0], ] ) -skip_no_data = pytest.mark.skipif( - not path.exists(DATA), - reason="""\ -Cannot find Ab_PET_mMR_test in ${DATA_ROOT:-~} (%s). -Get it from https://zenodo.org/record/3877529 -""" - % HOME, -) -no_matlab_warn = pytest.mark.filterwarnings( - "ignore:.*collections.abc:DeprecationWarning" -) +no_matlab_warn = mark.filterwarnings("ignore:.*collections.abc:DeprecationWarning") +no_scipy_warn = mark.filterwarnings("ignore:numpy.ufunc size changed.*:RuntimeWarning") def assert_equal_arrays(x, y, nmse_tol=0, denan=True): @@ -60,25 +39,23 @@ def assert_equal_arrays(x, y, nmse_tol=0, denan=True): ) -@skip_no_data +@no_scipy_warn @no_matlab_warn -def test_resample(): - with tmpdir() as outpath: - res = regseg.resample_spm(PET, MRI, MRI2PET, outpath=outpath) - res = nii.getnii(res) +def test_resample(PET, MRI, tmp_path): + res = regseg.resample_spm(PET, MRI, MRI2PET, outpath=tmp_path / "resample") + res = nii.getnii(res) ref = nii.getnii(PET) assert res.shape == ref.shape assert not np.isnan(res).all() -@skip_no_data +@no_scipy_warn @no_matlab_warn -def test_coreg(): - with tmpdir() as outpath: - res = regseg.coreg_spm(PET, MRI, outpath=outpath) - assert_equal_arrays(res["affine"], MRI2PET, 1e-4) - - with tmpdir() as outpath: - res = regseg.resample_spm(PET, MRI, res["affine"], outpath=outpath) - ref = regseg.resample_spm(PET, MRI, MRI2PET, outpath=outpath) - assert_equal_arrays(nii.getnii(res), nii.getnii(ref), 1e-4) +def test_coreg(PET, MRI, tmp_path): + res = regseg.coreg_spm(PET, MRI, outpath=tmp_path / "coreg") + assert_equal_arrays(res["affine"], MRI2PET, 5e-4) + + outpath = tmp_path / "resamp" + res = regseg.resample_spm(PET, MRI, res["affine"], outpath=outpath) + ref = regseg.resample_spm(PET, MRI, MRI2PET, outpath=outpath) + assert_equal_arrays(nii.getnii(res), nii.getnii(ref), 1e-4)