Retrieve the bidsmap file in a less fuzzy / more robust way (github i…

…ssue #221)

bidscoin/bcoin.py

@@ -463,11 +463,8 @@ def test_bidsmap(bidsmapfile: str):
 
     bidsmapfile = Path(bidsmapfile)
     if bidsmapfile.is_dir():
-        bidsfolder  = bidsmapfile/'code'/'bidscoin'
-        bidsmapfile = Path()
-    else:
-        bidsfolder  = Path()
-    bidsmap, _ = bids.load_bidsmap(bidsmapfile, bidsfolder, checks=(True, True, True))
+        bidsmapfile = bidsmapfile/'code'/'bidscoin'/'bidsmap.yaml'
+    bidsmap, _ = bids.load_bidsmap(bidsmapfile, checks=(True, True, True))
 
     return bids.validate_bidsmap(bidsmap, 1)
 

bidscoin/bids.py

@@ -851,47 +851,35 @@ def get_p7field(tagname: str, p7file: Path) -> Union[str, int]:
 # ---------------- All function below this point are bidsmap related. TODO: make a class out of them -------------------
 
 
-def load_bidsmap(yamlfile: Path, folder: Path=Path(), plugins:Union[tuple,list]=(), checks: Tuple[bool, bool, bool]=(True, True, True)) -> Tuple[dict, Path]:
+def load_bidsmap(yamlfile: Path=Path(), folder: Path=templatefolder, plugins:Union[tuple,list]=(), checks: Tuple[bool, bool, bool]=(True, True, True)) -> Tuple[dict, Path]:
     """
     Read the mapping heuristics from the bidsmap yaml-file. If yamlfile is not fullpath, then 'folder' is first searched before
     the default 'heuristics'. If yamfile is empty, then first 'bidsmap.yaml' is searched for, then 'bidsmap_template'. So fullpath
     has precedence over folder and bidsmap.yaml has precedence over the bidsmap_template.
 
     NB: A run['datasource'] = DataSource object is added to every run-item
 
-    :param yamlfile:    The full pathname or basename of the bidsmap yaml-file. If None, the default bidsmap_template file in the heuristics folder is used
-    :param folder:      Only used when yamlfile=basename or None: yamlfile is then first searched for in folder and then falls back to the ./heuristics folder (useful for centrally managed template yaml-files)
+    :param yamlfile:    The full pathname or basename of the bidsmap yaml-file
+    :param folder:      Only used when yamlfile=basename or None: yamlfile is then assumed to be in folder
     :param plugins:     List of plugins to be used (with default options, overrules the plugin list in the study/template bidsmaps). Leave empty to use all plugins in the bidsmap
     :param checks:      Booleans to check if all (bidskeys, bids-suffixes, bids-values) in the run are present according to the BIDS schema specifications
     :return:            Tuple with (1) ruamel.yaml dict structure, with all options, BIDS mapping heuristics, labels and attributes, etc. and (2) the fullpath yaml-file
     """
 
     # Input checking
-    if not folder.name or not folder.is_dir():
-        folder = templatefolder
     if not yamlfile.name:
-        yamlfile = folder/'bidsmap.yaml'
-        if not yamlfile.is_file():
-            yamlfile = bidsmap_template
-
-    # Add a standard file-extension if needed
+        yamlfile = Path('bidsmap.yaml')
     if not yamlfile.suffix:
-        yamlfile = yamlfile.with_suffix('.yaml')
-
-    # Get the full path to the bidsmap yaml-file
+        yamlfile = yamlfile.with_suffix('.yaml')    # Add a standard file-extension if needed
     if len(yamlfile.parents) == 1:
-        if (folder/yamlfile).is_file():
-            yamlfile = folder/yamlfile
-        else:
-            yamlfile = templatefolder/yamlfile
-
+        yamlfile = folder/yamlfile                  # Get the full path to the bidsmap yaml-file
     if not yamlfile.is_file():
         LOGGER.verbose(f"No existing bidsmap file found: {yamlfile}")
         return {}, yamlfile
-    elif any(checks):
-        LOGGER.info(f"Reading: {yamlfile}")
 
     # Read the heuristics from the bidsmap file
+    if any(checks):
+        LOGGER.info(f"Reading: {yamlfile}")
     with yamlfile.open('r') as stream:
         bidsmap = yaml.load(stream)
 

bidscoin/bidseditor.py

@@ -1498,7 +1498,7 @@ def export_run(self):
         if yamlfile:
             LOGGER.info(f'Exporting run item: bidsmap[{self.dataformat}][{self.target_datatype}] -> {yamlfile}')
             yamlfile   = Path(yamlfile)
-            bidsmap, _ = bids.load_bidsmap(yamlfile, Path(), checks=(False, False, False))
+            bidsmap, _ = bids.load_bidsmap(yamlfile, checks=(False, False, False))
             bids.append_run(bidsmap, self.target_run)
             bids.save_bidsmap(yamlfile, bidsmap)
             QMessageBox.information(self, 'Edit BIDS mapping', f"Successfully exported:\n\nbidsmap[{self.dataformat}][{self.target_datatype}] -> {yamlfile}")
@@ -1647,7 +1647,7 @@ def seteditable(self, iseditable: bool=True):
             self.setForeground(QtGui.QColor('gray'))
 
 
-def bidseditor(bidsfolder: str, bidsmapfile: str='', templatefile: str='') -> None:
+def bidseditor(bidsfolder: str, bidsmapfile: str='', templatefile: str=bidsmap_template) -> None:
     """
     Collects input and launches the bidseditor GUI
 
@@ -1667,7 +1667,7 @@ def bidseditor(bidsfolder: str, bidsmapfile: str='', templatefile: str='') -> No
     LOGGER.info(f">>> bidseditor bidsfolder={bidsfolder} bidsmap={bidsmapfile} template={templatefile}")
 
     # Obtain the initial bidsmap info
-    template_bidsmap, templatefile = bids.load_bidsmap(templatefile, bidsfolder/'code'/'bidscoin', checks=(True, True, False))
+    template_bidsmap, templatefile = bids.load_bidsmap(templatefile, checks=(True, True, False))
     input_bidsmap, bidsmapfile     = bids.load_bidsmap(bidsmapfile,  bidsfolder/'code'/'bidscoin')
     if input_bidsmap.get('Options'):
         template_bidsmap['Options'] = input_bidsmap['Options']      # Always use the options of the input bidsmap

bidscoin/bidsmapper.py

@@ -71,7 +71,7 @@ def bidsmapper(rawfolder: str, bidsfolder: str, bidsmapfile: str, templatefile:
 
     # Get the heuristics for filling the new bidsmap (NB: plugins are stored in the bidsmaps)
     bidsmap_old, bidsmapfile = bids.load_bidsmap(bidsmapfile,  bidscoinfolder, plugins)
-    template, _              = bids.load_bidsmap(templatefile, bidscoinfolder, plugins, checks=(True, True, False))
+    template, _              = bids.load_bidsmap(templatefile, plugins=plugins, checks=(True, True, False))
 
     # Create the new bidsmap as a copy / bidsmap skeleton with no data type entries (i.e. bidsmap with empty lists)
     if force and bidsmapfile.is_file():
@@ -99,7 +99,6 @@ def bidsmapper(rawfolder: str, bidsfolder: str, bidsmapfile: str, templatefile:
     # Start with an empty skeleton if we didn't have an old bidsmap
     if not bidsmap_old:
         bidsmap_old = copy.deepcopy(bidsmap_new)
-        bidsmapfile = bidscoinfolder/'bidsmap.yaml'
 
     # Import the data scanning plugins
     plugins = [bcoin.import_plugin(plugin, ('bidsmapper_plugin',)) for plugin in bidsmap_new['Options']['plugins']]

bidscoin/utilities/bidsparticipants.py

@@ -84,7 +84,7 @@ def bidsparticipants(rawfolder: str, bidsfolder: str, keys: list, bidsmapfile: s
     LOGGER.info(f">>> bidsparticipants sourcefolder={rawfolder} bidsfolder={bidsfolder} bidsmap={bidsmapfile}")
 
     # Get the bidsmap sub-/ses-prefix from the bidsmap YAML-file
-    bidsmap,_ = bids.load_bidsmap(Path(bidsmapfile), bidsfolder /'code' /'bidscoin', checks=(False, False, False))
+    bidsmap,_ = bids.load_bidsmap(Path(bidsmapfile), bidsfolder/'code'/'bidscoin', checks=(False, False, False))
     if not bidsmap:
         LOGGER.info('Make sure to run "bidsmapper" first, exiting now')
         return

docs/CHANGELOG.md

@@ -15,10 +15,11 @@
 - Option to exclude datatypes from being saved in bids/derivatives
 
 ### Changed
-- `bidscoiner_plugin()` API: you can just return a personals dict instead of writing it to `participants.tsv`
+- `bidscoiner_plugin()` API: you can (should) return a personals dict (instead of writing it to `participants.tsv`) and the datasource targets
 - Using DRMAA library for skullstrip (instead of qsub/sbatch)
 - Removed the pet2bids and phys2bids plugins (code is no longer actively developed)
-- Sorting of DICOMDIR files
+- Sorting of DICOMDIR files is more robust
+- Retrieving the bidsmap yaml-file from the user argument is less fuzzy
 
 ## [4.2.1] - 2023-10-30
 

tests/test_bids.py

@@ -8,7 +8,7 @@
 from pathlib import Path
 from nibabel.testing import data_path
 from pydicom.data import get_testdata_file
-from bidscoin import bcoin, bids, bidsmap_template
+from bidscoin import bcoin, bids
 
 bcoin.setup_logging()
 
@@ -29,7 +29,7 @@ def par_file():
 
 
 @pytest.fixture(scope='module')
-def test_bidsmap():
+def study_bidsmap():
     """The path to the study bidsmap `test_data/bidsmap.yaml`"""
     return Path(__file__).parent/'test_data'/'bidsmap.yaml'
 
@@ -152,28 +152,37 @@ def test_match_runvalue():
     assert bids.match_runvalue(r'\[1, 2, 3\]',  [1, 2, 3])             == False
 
 
-def test_load_bidsmap(test_bidsmap):
+def test_load_bidsmap(study_bidsmap):
 
-    # Test loading with recommended arguments for load_bidsmap
-    full_arguments_map, return_path = bids.load_bidsmap(Path(test_bidsmap.name), test_bidsmap.parent)
-    assert type(full_arguments_map) == ruamel.yaml.comments.CommentedMap
-    assert full_arguments_map is not []
+    # Test loading with standard arguments for load_bidsmap
+    bidsmap, filepath = bids.load_bidsmap(Path(study_bidsmap.name), study_bidsmap.parent)
+    assert type(bidsmap) == ruamel.yaml.comments.CommentedMap
+    assert bidsmap != {}
+    assert filepath == study_bidsmap
+    assert bidsmap['DICOM']['anat'][0]['provenance'] == '/Users/galassiae/Projects/bidscoin/bidscointutorial/raw/sub-001/ses-01/007-t1_mprage_sag_ipat2_1p0iso/00001_1.3.12.2.1107.5.2.43.66068.2020042808523182387402502.IMA'
 
-    # Test loading with no input folder0, should load default from heuristics folder
-    no_input_folder_map, _ = bids.load_bidsmap(test_bidsmap)
-    assert type(no_input_folder_map) == ruamel.yaml.comments.CommentedMap
-    assert no_input_folder_map is not []
+    # Test loading with fullpath argument
+    bidsmap, _ = bids.load_bidsmap(study_bidsmap)
+    assert type(bidsmap) == ruamel.yaml.comments.CommentedMap
+    assert bidsmap != {}
+    assert bidsmap['DICOM']['anat'][0]['provenance'] == '/Users/galassiae/Projects/bidscoin/bidscointutorial/raw/sub-001/ses-01/007-t1_mprage_sag_ipat2_1p0iso/00001_1.3.12.2.1107.5.2.43.66068.2020042808523182387402502.IMA'
 
-    # Test loading with full path to only bidsmap file
-    full_path_to_bidsmap_map, _ = bids.load_bidsmap(test_bidsmap)
-    assert type(full_path_to_bidsmap_map) == ruamel.yaml.comments.CommentedMap
-    assert no_input_folder_map is not []
+    # Test loading with standard argument for the template bidsmap
+    bidsmap, _ = bids.load_bidsmap(Path('bidsmap_dccn'))
+    assert type(bidsmap) == ruamel.yaml.comments.CommentedMap
+    assert bidsmap != {}
+    assert bidsmap['DICOM']['anat'][0]['provenance'] == 'sub--unknown/ses--unknown/DICOM_anat_id001'
 
+    # Test loading with a dummy argument
+    bidsmap, filepath = bids.load_bidsmap(Path('dummy'))
+    assert bidsmap  == {}
+    assert filepath == bids.templatefolder/'dummy.yaml'
 
-def test_validate_bidsmap(test_bidsmap):
+
+def test_validate_bidsmap(study_bidsmap):
 
     # Load a BIDS-valid study bidsmap
-    bidsmap, _ = bids.load_bidsmap(test_bidsmap)
+    bidsmap, _ = bids.load_bidsmap(study_bidsmap)
     run        = bidsmap['DICOM']['func'][0]
     assert bids.validate_bidsmap(bidsmap) == True
 
@@ -200,32 +209,32 @@ def test_validate_bidsmap(test_bidsmap):
     assert bids.validate_bidsmap(bidsmap) == False
 
 
-def test_check_bidsmap(test_bidsmap):
+def test_check_bidsmap(study_bidsmap):
 
     # Load a template and a study bidsmap
-    template_bidsmap, _ = bids.load_bidsmap(bidsmap_template, checks=(True, True, False))
-    study_bidsmap, _    = bids.load_bidsmap(test_bidsmap)
+    templatebidsmap, _ = bids.load_bidsmap(bids.bidsmap_template, checks=(True, True, False))
+    studybidsmap, _    = bids.load_bidsmap(study_bidsmap)
 
     # Test the output of the template bidsmap
     checks   = (True, True, False)
-    is_valid = bids.check_bidsmap(template_bidsmap, checks)
+    is_valid = bids.check_bidsmap(templatebidsmap, checks)
     for each, check in zip(is_valid, checks):
         assert each in (None, True, False)
         if check:
             assert each in (None, True)
 
     # Test the output of the study bidsmap
-    is_valid = bids.check_bidsmap(study_bidsmap, checks)
+    is_valid = bids.check_bidsmap(studybidsmap, checks)
     for each, check in zip(is_valid, checks):
         assert each in (None, True, False)
         if check:
             assert each == True
 
 
-def test_check_run(test_bidsmap):
+def test_check_run(study_bidsmap):
 
     # Load a bidsmap
-    bidsmap, _ = bids.load_bidsmap(test_bidsmap)
+    bidsmap, _ = bids.load_bidsmap(study_bidsmap)
 
     # Collect the first func run-item
     checks = (True, True, True)             # = (keys, suffixes, values)
@@ -273,10 +282,10 @@ def test_check_ignore():
     assert bids.check_ignore('sub-01_foo.nii',     bidsignore, 'file') == True
 
 
-def test_find_run(test_bidsmap):
+def test_find_run(study_bidsmap):
 
     # Load a bidsmap and create a duplicate dataformat section
-    bidsmap, _     = bids.load_bidsmap(test_bidsmap)
+    bidsmap, _     = bids.load_bidsmap(study_bidsmap)
     bidsmap['PET'] = copy.deepcopy(bidsmap['DICOM'])
 
     # Collect provenance of the first anat run-item
@@ -303,10 +312,10 @@ def test_find_run(test_bidsmap):
     assert run.get('provenance') == tag
 
 
-def test_delete_run(test_bidsmap):
+def test_delete_run(study_bidsmap):
 
     # Load a study bidsmap and delete one anat run
-    bidsmap, _ = bids.load_bidsmap(test_bidsmap)
+    bidsmap, _ = bids.load_bidsmap(study_bidsmap)
     nritems    = len(bidsmap['DICOM']['anat'])
     provenance = bidsmap['DICOM']['anat'][0]['provenance']
     bids.delete_run(bidsmap, provenance)
@@ -315,10 +324,10 @@ def test_delete_run(test_bidsmap):
     assert bids.find_run(bidsmap, provenance) == {}
 
 
-def test_append_run(test_bidsmap):
+def test_append_run(study_bidsmap):
 
     # Load a study bidsmap and delete one anat run
-    bidsmap, _ = bids.load_bidsmap(test_bidsmap)
+    bidsmap, _ = bids.load_bidsmap(study_bidsmap)
 
     # Collect and modify the first anat run-item
     run                          = copy.deepcopy(bidsmap['DICOM']['anat'][0])
@@ -330,10 +339,10 @@ def test_append_run(test_bidsmap):
     assert Path(bidsmap['Foo']['Bar'][0]['provenance']) == Path(run['provenance'])
 
 
-def test_update_bidsmap(test_bidsmap):
+def test_update_bidsmap(study_bidsmap):
 
     # Load a study bidsmap and move the first run-item from func to anat
-    bidsmap, _ = bids.load_bidsmap(test_bidsmap)
+    bidsmap, _ = bids.load_bidsmap(study_bidsmap)
 
     # Collect and modify the first func run-item
     run                        = copy.deepcopy(bidsmap['DICOM']['func'][0])
@@ -350,10 +359,10 @@ def test_update_bidsmap(test_bidsmap):
     assert bidsmap['DICOM']['anat'][-1]['bids']['foo'] == 'bar'
 
 
-def test_exist_run(test_bidsmap):
+def test_exist_run(study_bidsmap):
 
     # Load a bidsmap
-    bidsmap, _ = bids.load_bidsmap(test_bidsmap)
+    bidsmap, _ = bids.load_bidsmap(study_bidsmap)
 
     # Collect the first anat run-item
     run = copy.deepcopy(bidsmap['DICOM']['anat'][0])