First working version to implement Github PR #19

bidscoin/__init__.py

@@ -14,15 +14,67 @@
 SPDX-License-Identifier: GPL-3.0-or-later
 """
 
+import urllib.request
+import json
 from pathlib import Path
-from importlib.metadata import version
+from importlib.metadata import version as libversion
+from typing import Tuple, Union
 try:
     import tomllib
 except ModuleNotFoundError:
     import tomli as tomllib
+try:
+    from .due import due, Doi
+except ImportError:
+    from due import due, Doi
+
+# Add the BIDScoin citation
+due.cite(Doi('10.3389/fninf.2021.770608'), description='A toolkit to convert source data to the Brain Imaging Data Structure (BIDS)', path='bidscoin')
 
+# Get the BIDScoin version
 try:
-    __version__ = version('bidscoin')
+    __version__ = libversion('bidscoin')
 except Exception:
     with open(Path(__file__).parents[1]/'pyproject.toml', 'rb') as fid:
         __version__ = tomllib.load(fid)['project']['version']
+
+# Define the default paths
+tutorialurl      = 'https://surfdrive.surf.nl/files/index.php/s/HTxdUbykBZm2cYM/download'
+bidscoinfolder   = Path(__file__).parent
+schemafolder     = bidscoinfolder/'schema'
+heuristicsfolder = bidscoinfolder/'heuristics'
+pluginfolder     = bidscoinfolder/'plugins'
+bidsmap_template = heuristicsfolder/'bidsmap_dccn.yaml'     # Default template bidsmap TODO: make it a user setting (in $HOME)?
+
+
+def version(check: bool=False) -> Union[str, Tuple]:
+    """
+    Reads the BIDSCOIN version from the VERSION.TXT file and from pypi
+
+    :param check:   Check if the current version is up-to-date
+    :return:        The version number or (version number, checking message) if check=True
+    """
+
+    # Check pypi for the latest version number
+    if check:
+        try:
+            stream      = urllib.request.urlopen('https://pypi.org/pypi/bidscoin/json').read()
+            pypiversion = json.loads(stream)['info']['version']
+        except Exception as pypierror:
+            return __version__, "(Could not check for new BIDScoin versions)"
+        if __version__ != pypiversion:
+            return __version__, f"NB: Your BIDScoin version is NOT up-to-date: {__version__} -> {pypiversion}"
+        else:
+            return __version__, "Your BIDScoin version is up-to-date :-)"
+
+    return __version__
+
+
+def bidsversion() -> str:
+    """
+    Reads the BIDS version from the BIDS_VERSION.TXT file
+
+    :return:    The BIDS version number
+    """
+
+    return (schemafolder/'BIDS_VERSION').read_text().strip()

bidscoin/bcoin.py

@@ -1,57 +1,32 @@
 #!/usr/bin/env python3
 """
-BIDScoin is a toolkit to convert and organize raw data-sets according to the Brain Imaging Data Structure (BIDS)
+A BIDScoin library and application with utilities to perform generic management tasks (See also cli/_bcoin.py)
 
-The basic workflow is to run these two tools:
-
-  $ bidsmapper sourcefolder bidsfolder        # This produces a study bidsmap and launches a GUI
-  $ bidscoiner sourcefolder bidsfolder        # This converts your data to BIDS according to the study bidsmap
-
-Set the environment variable BIDSCOIN_DEBUG=TRUE in your console to run BIDScoin in its more verbose DEBUG logging mode
-
-For more documentation see: https://bidscoin.readthedocs.io
+@author: Marcel Zwiers
 """
 
-import argparse
 import coloredlogs
 import inspect
-import json
 import logging
 import os
 import shutil
 import subprocess
 import sys
-import textwrap
 import urllib.request
 from functools import lru_cache
-from importlib.metadata import entry_points, version as libversion
+from importlib.metadata import entry_points
 from importlib.util import spec_from_file_location, module_from_spec
 from pathlib import Path
 from typing import Tuple, Union, List
 from ruamel.yaml import YAML
 from tqdm import tqdm
-try:
-    import tomllib
-except ModuleNotFoundError:
-    import tomli as tomllib
-try:
-    from .due import due, Doi
-except ImportError:
-    from due import due, Doi
+from importlib.util import find_spec
+if find_spec('bidscoin') is None:
+    sys.path.append(str(Path(__file__).parents[1]))
+from bidscoin import heuristicsfolder, pluginfolder, bidsmap_template, tutorialurl
 
 yaml = YAML()
 
-# Add the BIDScoin citation
-due.cite(Doi('10.3389/fninf.2021.770608'), description='A toolkit to convert source data to the Brain Imaging Data Structure (BIDS)', path='bidscoin')
-
-# Define the default paths
-tutorialurl      = 'https://surfdrive.surf.nl/files/index.php/s/HTxdUbykBZm2cYM/download'
-bidscoinfolder   = Path(__file__).parent
-schemafolder     = bidscoinfolder/'schema'
-heuristicsfolder = bidscoinfolder/'heuristics'
-pluginfolder     = bidscoinfolder/'plugins'
-bidsmap_template = heuristicsfolder/'bidsmap_dccn.yaml'     # Default template bidsmap TODO: make it a user setting (in $HOME)?
-
 # Get the BIDSCOIN_DEBUG environment variable to set the log-messages and logging level, etc
 debug  = os.environ.get('BIDSCOIN_DEBUG')
 debug  = True if debug and debug.upper() not in ('0', 'FALSE', 'N', 'NO', 'NONE') else False
@@ -182,46 +157,6 @@ def reporterrors() -> str:
     return errors
 
 
-def version(check: bool=False) -> Union[str, Tuple]:
-    """
-    Reads the BIDSCOIN version from the VERSION.TXT file and from pypi
-
-    :param check:   Check if the current version is up-to-date
-    :return:        The version number or (version number, checking message) if check=True
-    """
-
-    try:
-        localversion = libversion('bidscoin')
-    except Exception:
-        with open(Path(__file__).parents[1]/'pyproject.toml', 'rb') as fid:
-            localversion = tomllib.load(fid)['project']['version']
-
-    # Check pypi for the latest version number
-    if check:
-        try:
-            stream      = urllib.request.urlopen('https://pypi.org/pypi/bidscoin/json').read()
-            pypiversion = json.loads(stream)['info']['version']
-        except Exception as pypierror:
-            LOGGER.info(f"Checking BIDScoin version on https://pypi.org/pypi/bidscoin failed:\n{pypierror}")
-            return localversion, "(Could not check for new BIDScoin versions)"
-        if localversion != pypiversion:
-            return localversion, f"NB: Your BIDScoin version is NOT up-to-date: {localversion} -> {pypiversion}"
-        else:
-            return localversion, "Your BIDScoin version is up-to-date :-)"
-
-    return localversion
-
-
-def bidsversion() -> str:
-    """
-    Reads the BIDS version from the BIDSVERSION.TXT file
-
-    :return:    The BIDS version number
-    """
-
-    return (schemafolder/'BIDS_VERSION').read_text().strip()
-
-
 def run_command(command: str) -> int:
     """
     Runs a command in a shell using subprocess.run(command, ..)
@@ -654,28 +589,12 @@ def pulltutorialdata(tutorialfolder: str) -> None:
 def main():
     """Console script usage"""
 
+    from bidscoin.cli._bcoin import get_parser
+
     setup_logging()
-    localversion, versionmessage = version(check=True)
 
     # Parse the input arguments and run bidscoiner(args)
-    parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter,
-                                     description=textwrap.dedent(__doc__),
-                                     epilog='examples:\n'
-                                            '  bidscoin -l\n'
-                                            '  bidscoin -d data/bidscoin_tutorial\n'
-                                            '  bidscoin -t\n'
-                                            '  bidscoin -t my_template_bidsmap\n'
-                                            '  bidscoin -b my_study_bidsmap\n'
-                                            '  bidscoin -i data/my_template_bidsmap.yaml downloads/my_plugin.py\n ')
-    parser.add_argument('-l', '--list',        help='List all executables (i.e. the apps, bidsapps and utilities)', action='store_true')
-    parser.add_argument('-p', '--plugins',     help='List all installed plugins and template bidsmaps', action='store_true')
-    parser.add_argument('-i', '--install',     help='A list of template bidsmaps and/or bidscoin plugins to install', nargs='+')
-    parser.add_argument('-u', '--uninstall',   help='A list of template bidsmaps and/or bidscoin plugins to uninstall', nargs='+')
-    parser.add_argument('-d', '--download',    help='Download folder. If given, tutorial MRI data will be downloaded here')
-    parser.add_argument('-t', '--test',        help='Test the bidscoin installation and template bidsmap', nargs='?', const=bidsmap_template)
-    parser.add_argument('-b', '--bidsmaptest', help='Test the run-items and their bidsnames of all normal runs in the study bidsmap. Provide the bids-folder or the bidsmap filepath')
-    parser.add_argument('-v', '--version',     help='Show the installed version and check for updates', action='version', version=f"BIDS-version:\t\t{bidsversion()}\nBIDScoin-version:\t{localversion}, {versionmessage}")
-    args = parser.parse_args(None if sys.argv[1:] else ['--help'])
+    args = get_parser().parse_args(None if sys.argv[1:] else ['--help'])
 
     list_executables(show=args.list)
     list_plugins(show=args.plugins)

bidscoin/bids.py

@@ -1,5 +1,5 @@
 """
-Module with helper functions
+BIDScoin module with bids/bidsmap related helper functions
 
 Some functions are derived from dac2bids.py from Daniel Gomez 29.08.2016
 https://github.com/dangom/dac2bids/blob/master/dac2bids.py
@@ -21,33 +21,31 @@
 from typing import Union, List, Tuple
 from nibabel.parrec import parse_PAR_header
 from pydicom import dcmread, fileset, datadict
-try:
-    from bidscoin import bcoin
-    from bidscoin.utilities import dicomsort
-except ImportError:
+from importlib.util import find_spec
+if find_spec('bidscoin') is None:
     import sys
-    sys.path.append(str(Path(__file__).parent/'utilities'))                     # This should work if bidscoin was not pip-installed
-    import bcoin
-    import dicomsort
+    sys.path.append(str(Path(__file__).parents[1]))
+from bidscoin import bcoin, schemafolder, __version__
+from bidscoin.utilities import dicomsort
 from ruamel.yaml import YAML
 yaml = YAML()
 
 LOGGER = logging.getLogger(__name__)
 
 # Read the BIDS schema data
-with (bcoin.schemafolder/'objects'/'datatypes.yaml').open('r') as _stream:
+with (schemafolder/'objects'/'datatypes.yaml').open('r') as _stream:
     bidsdatatypesdef = yaml.load(_stream)                                       # The valid BIDS datatypes, along with their full names and descriptions
 datatyperules = {}
-for _datatypefile in (bcoin.schemafolder/'rules'/'files'/'raw').glob('*.yaml'):
+for _datatypefile in (schemafolder/'rules'/'files'/'raw').glob('*.yaml'):
     with _datatypefile.open('r') as _stream:
         datatyperules[_datatypefile.stem] = yaml.load(_stream)                  # The entities that can/should be present for each BIDS datatype
-with (bcoin.schemafolder/'objects'/'suffixes.yaml').open('r') as _stream:
+with (schemafolder/'objects'/'suffixes.yaml').open('r') as _stream:
     suffixes = yaml.load(_stream)                                               # The descriptions of the valid BIDS file suffixes
-with (bcoin.schemafolder/'objects'/'entities.yaml').open('r') as _stream:
+with (schemafolder/'objects'/'entities.yaml').open('r') as _stream:
     entities = yaml.load(_stream)                                               # The descriptions of the entities present in BIDS filenames
-with (bcoin.schemafolder/'rules'/'entities.yaml').open('r') as _stream:
+with (schemafolder/'rules'/'entities.yaml').open('r') as _stream:
     entitiesorder = yaml.load(_stream)                                          # The order in which the entities should appear within filenames
-with (bcoin.schemafolder/'objects'/'metadata.yaml').open('r') as _stream:
+with (schemafolder/'objects'/'metadata.yaml').open('r') as _stream:
     metafields = yaml.load(_stream)                                             # The descriptions of the valid BIDS metadata fields
 
 
@@ -884,10 +882,10 @@ def load_bidsmap(yamlfile: Path, folder: Path=Path(), plugins:Union[tuple,list]=
         bidsmapversion = bidsmap['Options']['version']
     else:
         bidsmapversion = 'Unknown'
-    if bidsmapversion.rsplit('.', 1)[0] != bcoin.version().rsplit('.', 1)[0] and any(check):
-        LOGGER.warning(f'BIDScoiner version conflict: {yamlfile} was created with version {bidsmapversion}, but this is version {bcoin.version()}')
-    elif bidsmapversion != bcoin.version() and any(check):
-        LOGGER.info(f'BIDScoiner version difference: {yamlfile} was created with version {bidsmapversion}, but this is version {bcoin.version()}. This is normally ok but check the https://bidscoin.readthedocs.io/en/latest/CHANGELOG.html')
+    if bidsmapversion.rsplit('.', 1)[0] != __version__.rsplit('.', 1)[0] and any(check):
+        LOGGER.warning(f'BIDScoiner version conflict: {yamlfile} was created with version {bidsmapversion}, but this is version {__version__}')
+    elif bidsmapversion != __version__ and any(check):
+        LOGGER.info(f'BIDScoiner version difference: {yamlfile} was created with version {bidsmapversion}, but this is version {__version__}. This is normally ok but check the https://bidscoin.readthedocs.io/en/latest/CHANGELOG.html')
 
     # Make sure we get a proper plugin options and dataformat sections (use plugin default bidsmappings when a template bidsmap is loaded)
     if not bidsmap['Options'].get('plugins'):
@@ -1213,14 +1211,14 @@ def check_run(datatype: str, run: dict, check: Tuple[bool, bool, bool]=(False, F
     return run_keysok, run_suffixok, run_valsok
 
 
-def check_ignore(entry: str, bidsignore: Union[str,list], type: str='dir') -> bool:
+def check_ignore(entry: str, bidsignore: Union[str,list], datatype: str= 'dir') -> bool:
     """
     A rudimentary check whether `entry` should be BIDS-ignored. This function should eventually be replaced by bids_validator functionality
     See also https://github.com/bids-standard/bids-specification/issues/131
 
     :param entry:       The entry that is checked against the bidsignore (e.g. a directory/datatype such as `anat` or a file such as `sub-001_ct.nii.gz`)
     :param bidsignore:  The list or semicolon separated bidsignore pattern (e.g. from the bidscoin Options such as `mrs/;extra_data/;sub-*_ct.*`)
-    :param type:        The entry type, i.e. 'dir' or 'file', that can be used to limit the check
+    :param datatype:    The entry datatype, i.e. 'dir' or 'file', that can be used to limit the check
     :return:            True if the entry should be ignored, else False
     """
 
@@ -1238,8 +1236,8 @@ def check_ignore(entry: str, bidsignore: Union[str,list], type: str='dir') -> bo
 
     ignore = False
     for item in bidsignore:
-        if type == 'dir' and not item.endswith('/'): continue
-        if type == 'file'    and item.endswith('/'): continue
+        if datatype =='dir' and not item.endswith('/'): continue
+        if datatype =='file'    and item.endswith('/'): continue
         if item.endswith('/'):
             item = item[0:-1]
         if fnmatch.fnmatch(entry, item):

bidscoin/bidsapps/deface.py

@@ -1,32 +1,20 @@
 #!/usr/bin/env python3
-"""
-A wrapper around the 'pydeface' defacing tool (https://github.com/poldracklab/pydeface).
-
-Except for BIDS inheritances and IntendedFor usage, this wrapper is BIDS-aware (a 'bidsapp')
-and writes BIDS compliant output
-
-Linux users can distribute the computations to their HPC compute cluster if the DRMAA
-libraries are installed and the DRMAA_LIBRARY_PATH environment variable set
-
-For multi-echo data see `medeface`
-"""
+"""A bidsapp that wraps around the 'pydeface' defacing tool (See also cli/_deface.py)"""
 
 import os
 import shutil
-import argparse
 import json
 import logging
 import pandas as pd
 import pydeface.utils as pdu
 from tqdm import tqdm
 from tqdm.contrib.logging import logging_redirect_tqdm
 from pathlib import Path
-try:
-    from bidscoin import bcoin, bids
-except ImportError:
+from importlib.util import find_spec
+if find_spec('bidscoin') is None:
     import sys
-    sys.path.append(str(Path(__file__).parents[1]))             # This should work if bidscoin was not pip-installed
-    import bcoin, bids
+    sys.path.append(str(Path(__file__).parents[2]))
+from bidscoin import bcoin, bids
 
 
 def deface(bidsdir: str, pattern: str, subjects: list, force: bool, output: str, cluster: bool, nativespec: str, kwargs: dict):
@@ -162,25 +150,9 @@ def deface(bidsdir: str, pattern: str, subjects: list, force: bool, output: str,
 def main():
     """Console script usage"""
 
-    class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter): pass
-
-    parser = argparse.ArgumentParser(formatter_class=CustomFormatter,
-                                     description=__doc__,
-                                     epilog='examples:\n'
-                                            '  deface myproject/bids anat/*_T1w*\n'
-                                            '  deface myproject/bids anat/*_T1w* -p 001 003 -o derivatives\n'
-                                            '  deface myproject/bids anat/*_T1w* -c -n "-l walltime=00:60:00,mem=4gb"\n'
-                                            '  deface myproject/bids anat/*_T1w* -a \'{"cost": "corratio", "verbose": ""}\'\n ')
-    parser.add_argument('bidsfolder',               help='The bids-directory with the subject data')
-    parser.add_argument('pattern',                  help="Globlike search pattern (relative to the subject/session folder) to select the images that need to be defaced, e.g. 'anat/*_T1w*'")
-    parser.add_argument('-p','--participant_label', help='Space separated list of sub-# identifiers to be processed (the sub-prefix can be left out). If not specified then all sub-folders in the bidsfolder will be processed', nargs='+')
-    parser.add_argument('-o','--output',            help=f"A string that determines where the defaced images are saved. It can be the name of a BIDS datatype folder, such as 'anat', or of the derivatives folder, i.e. 'derivatives'. If output is left empty then the original images are replaced by the defaced images")
-    parser.add_argument('-c','--cluster',           help='Use the DRMAA library to submit the deface jobs to a high-performance compute (HPC) cluster', action='store_true')
-    parser.add_argument('-n','--nativespec',        help='DRMAA native specifications for submitting deface jobs to the HPC cluster', default='-l walltime=00:30:00,mem=2gb')
-    parser.add_argument('-a','--args',              help='Additional arguments (in dict/json-style) that are passed to pydeface. See examples for usage', type=json.loads, default={})
-    parser.add_argument('-f','--force',             help='Deface all images, regardless if they have already been defaced (i.e. if {"Defaced": True} in the json sidecar file)', action='store_true')
-    args = parser.parse_args()
+    from bidscoin.cli._deface import get_parser
 
+    args = get_parser().parse_args()
     deface(bidsdir    = args.bidsfolder,
            pattern    = args.pattern,
            subjects   = args.participant_label,