Add exceptions BIDS derivatives (GitHub issue #218)

bidscoin/bids.py

@@ -1722,15 +1722,17 @@ def get_matching_run(datasource: DataSource, bidsmap: dict, runtime=False) -> Tu
     return run_, False
 
 
-def get_derivatives(datatype: str) -> list:
+def get_derivatives(datatype: str, exceptions: tuple=()) -> list:
     """
     Retrieves a list of suffixes that are stored in the derivatives folder (e.g. the qMRI maps). TODO: Replace with a more systematic/documented method
     """
 
     if datatype == 'anat':
-        return [suffix for suffix in datatyperules[datatype]['parametric']['suffixes'] if suffix not in ('UNIT1',)]                                         # The qMRI data (maps)
+        return [suffix for suffix in datatyperules[datatype]['parametric']['suffixes']
+                if suffix not in exceptions + ('UNIT1',)]       # The qMRI data (maps)
     elif datatype == 'fmap':
-        return [suffix for typegroup in datatyperules[datatype] for suffix in datatyperules[datatype][typegroup]['suffixes'] if typegroup not in ('fieldmaps','pepolar')]     # The non-standard fmaps (file collections)
+        return [suffix for typegroup in datatyperules[datatype] for suffix in datatyperules[datatype][typegroup]['suffixes']
+                if suffix not in exceptions and typegroup not in ('fieldmaps','pepolar')]    # The non-standard fmaps (file collections)
     else:
         return []
 

bidscoin/bidseditor.py

@@ -46,15 +46,13 @@
     # self.ignoredatatypes : HELP_URL_DEFAULT
 }
 
-TOOLTIP_BIDSCOIN = """BIDScoin
-version:      Used to check for version conflicts 
-bidsignore:   List of data types that are added to the .bidsignore file,
-              e.g. extra_data/;myfile.txt;yourfile.csv
-subprefix:    The subject prefix used in the source data folders (e.g. "Pt" is the subprefix if subject folders are named "Pt018", "Pt019", ...)
-sesprefix:    The session prefix used in the source data folders (e.g. "M_" is the subprefix if session folders are named "M_pre", "M_post", ...)
-unknowntypes: Datatypes that are not part of BIDS but that are converted to a BIDS-like entries in the BIDS folder
-ignoretypes:  Datatypes that are excluded / not converted
-unzip:        Wildcard pattern to select tarball/zip-files in the sourcefolders that need to be unzipped (in a tempdir), e.g. '*.tar.gz'"""
+TOOLTIP_BIDSCOIN = f"""BIDScoin
+version:    Used to check for version conflicts 
+bidsignore: List of data types that are added to the .bidsignore file,
+            e.g. extra_data/;myfile.txt;yourfile.csv
+subprefix:  The subject prefix used in the source data folders (e.g. "Pt" is the subprefix if subject folders are named "Pt018", "Pt019", ...)
+sesprefix:  The session prefix used in the source data folders (e.g. "M_" is the subprefix if session folders are named "M_pre", "M_post", ...)
+For more information see: {MAIN_HELP_URL}/options.html"""
 
 TOOLTIP_DCM2NIIX = """dcm2niix2bids
 command: Command to run dcm2niix from the terminal, such as:
@@ -467,7 +465,8 @@ def update_subses_samples(self, output_bidsmap, dataformat):
                 subid, sesid = run['datasource'].subid_sesid(subid, sesid or '')
                 bidsname     = bids.get_bidsname(subid, sesid, run, not bids.check_ignore(datatype,self.bidsignore) and datatype not in self.ignoredatatypes)
                 ignore       = bids.check_ignore(datatype, self.bidsignore) or bids.check_ignore(bidsname+'.json', self.bidsignore, 'file')
-                if run['datasource'].dynamicvalue(run['bids']['suffix'], True, True) in bids.get_derivatives(datatype):
+                exceptions   = self.output_bidsmap['Options']['bidscoin'].get('notderivative',())
+                if run['datasource'].dynamicvalue(run['bids']['suffix'], True, True) in bids.get_derivatives(datatype, exceptions):
                     session  = self.bidsfolder/'derivatives'/'[manufacturer]'/subid/sesid
                 else:
                     session  = self.bidsfolder/subid/sesid

bidscoin/heuristics/bidsmap_dccn.yaml

@@ -29,6 +29,7 @@ Options:
     bidsignore: [mrs/, extra_data/, sub-*_ct.*]    # List of entries that are added to the .bidsignore file (for more info, see BIDS specifications), e.g. [extra_data/, pet/, myfile.txt, yourfile.csv]
     unknowntypes: [mrs, extra_data] # A list of datatypes that are converted to BIDS-like datatype folders
     ignoretypes: [exclude]          # A list of datatypes that are excluded / not converted to BIDS
+    notderivative: []               # A list of suffixes that should not be considered as a derivative datatype
     unzip:                          # Wildcard pattern to select tarball/zip-files in the source folders that need to be unzipped (in a tempdir) to expose the data, e.g. '*.tar.gz'
   plugins:                          # List of plugins with plugin-specific key-value pairs (that can be used by the plugin)
     dcm2niix2bids:                  # See dcm2niix -h and https://www.nitrc.org/plugins/mwiki/index.php/dcm2nii:MainPage#General_Usage for more info

bidscoin/heuristics/bidsmap_sst.yaml

@@ -28,6 +28,7 @@ Options:
     bidsignore: [mrs/, extra_data/, sub-*_ct.*]    # List of entries that are added to the .bidsignore file (for more info, see BIDS specifications), e.g. [extra_data/, pet/, myfile.txt, yourfile.csv]
     unknowntypes: [mrs, extra_data] # A list of datatypes that are converted to BIDS-like datatype folders
     ignoretypes: [exclude]          # A list of datatypes that are excluded / not converted to BIDS
+    notderivative: []               # A list of suffixes that should not be considered as a derivative datatype
     unzip:                          # Wildcard pattern to select tarball/zip-files in the source folders that need to be unzipped (in a tempdir) to expose the data, e.g. '*.tar.gz'
   plugins:                          # List of plugins with plugin-specific key-value pairs (that can be used by the plugin)
     dcm2niix2bids:                  # See dcm2niix -h and https://www.nitrc.org/plugins/mwiki/index.php/dcm2nii:MainPage#General_Usage for more info

bidscoin/heuristics/schema.json

@@ -22,6 +22,7 @@
                   "sesprefix": { "type": ["string", "null"] },
                   "unknowntypes": { "type": ["array"] },
                   "ignoretypes": { "type": ["array"] },
+                  "notderivative": { "type": ["array"] },
                   "unzip": { "type": ["string", "null"] }
                },
                "additionalProperties": false

bidscoin/plugins/dcm2niix2bids.py

@@ -122,7 +122,7 @@ def bidsmapper_plugin(session: Path, bidsmap_new: dict, bidsmap_old: dict, templ
         return
 
     # Collect the different DICOM/PAR source files for all runs in the session
-    sourcefiles = []
+    sourcefiles: List[Path] = []
     if dataformat == 'DICOM':
         for sourcedir in lsdirs(session, '**/*'):
             for n in range(1):      # Option: Use range(2) to scan two files and catch e.g. magnitude1/2 fieldmap files that are stored in one Series folder (but bidscoiner sees only the first file anyhow and it makes bidsmapper 2x slower :-()
@@ -208,8 +208,8 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> Union[None
         return
 
     # Make a list of all the data sources / runs
+    sources: List[Path] = []
     manufacturer = 'UNKNOWN'
-    sources      = []
     if dataformat == 'DICOM':
         sources      = lsdirs(session, '**/*')
         manufacturer = datasource.attributes('Manufacturer')
@@ -258,8 +258,9 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> Union[None
         matched_runs.append(run)
 
         # Create the BIDS session/datatype output folder
-        suffix = datasource.dynamicvalue(run['bids']['suffix'], True, True)
-        if suffix in bids.get_derivatives(datasource.datatype):
+        suffix     = datasource.dynamicvalue(run['bids']['suffix'], True, True)
+        exceptions = bidsmap['Options']['bidscoin'].get('notderivative', ())
+        if suffix in bids.get_derivatives(datasource.datatype, exceptions):
             outfolder = bidsfolder/'derivatives'/manufacturer.replace(' ','')/subid/sesid/datasource.datatype
         else:
             outfolder = bidsses/datasource.datatype
@@ -472,7 +473,7 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> Union[None
             outputfile = [file for file in jsonfile.parent.glob(jsonfile.stem + '.*') if file.suffix in ('.nii','.gz')]     # Find the corresponding NIfTI/tsv.gz file (there should be only one, let's not make assumptions about the .gz extension)
             if not outputfile:
                 LOGGER.exception(f"No data-file found with {jsonfile} when updating {scans_tsv}")
-            elif not bidsignore and not suffix in bids.get_derivatives(datasource.datatype):
+            elif not bidsignore and not suffix in bids.get_derivatives(datasource.datatype, exceptions):
                 acq_time = ''
                 if dataformat == 'DICOM':
                     acq_time = f"{datasource.attributes('AcquisitionDate')}T{datasource.attributes('AcquisitionTime')}"

bidscoin/plugins/spec2nii2bids.py

@@ -267,8 +267,9 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> Union[None
             json.dump(metadata, json_fid, indent=4)
 
         # Parse the acquisition time from the source header or else from the json file (NB: assuming the source file represents the first acquisition)
-        suffix = datasource.dynamicvalue(run['bids']['suffix'], True, True)
-        if not bidsignore and not suffix in bids.get_derivatives(datasource.datatype):
+        suffix     = datasource.dynamicvalue(run['bids']['suffix'], True, True)
+        exceptions = bidsmap['Options']['bidscoin'].get('notderivative', ())
+        if not bidsignore and not suffix in bids.get_derivatives(datasource.datatype, exceptions):
             acq_time = ''
             if dataformat == 'SPAR':
                 acq_time = datasource.attributes('scan_date')

docs/options.rst

@@ -23,7 +23,8 @@ These setting can be used by all the BIDScoin tools:
 - ``datatypes``: Datatypes that are converted to BIDS. This can be useful for ignoring / excluding specific datatypes (without changing their mappings)
 - ``unknowntypes``: Datatypes that are not part of BIDS but that are converted to a BIDS-like entries in the BIDS folder
 - ``ignoretypes``: Datatypes that are excluded / not converted"""
-- ``zip``: Wildcard pattern to select tarball/zip-files in the source folders that need to be unzipped (in a tempdir) to expose the data. Use for instance '\*.tar.gz' if your source data looks like ``sub-01\01_MPRAGE\dcmfiles.tar.gz``, etc
+- ``notderivative``: A list of suffixes that should not be considered as a derivative datatype (i.e. use this to make exceptions for what goes into ``bids/derivatives``)
+- ``unzip``: Wildcard pattern to select tarball/zip-files in the source folders that need to be unzipped (in a tempdir) to expose the data. Use for instance '\*.tar.gz' if your source data looks like ``sub-01\01_MPRAGE\dcmfiles.tar.gz``, etc
 
 The core working of BIDScoin and its plugins can be tested by clicking the corresponding [Test] button and inspection of the terminal output.