Move the meta-data handling over from the plugins to the bids library

bidscoin/bids.py

@@ -17,6 +17,7 @@
 import warnings
 import fnmatch
 import pandas as pd
+import ast
 from functools import lru_cache
 from pathlib import Path
 from typing import Union, List, Tuple
@@ -282,7 +283,7 @@ def dynamicvalue(self, value, cleanup: bool=True, runtime: bool=False):
         '<' and end with '>', but not with '<<' and '>>' unless runtime = True
 
         :param value:       The dynamic value that contains source attribute or filesystem property key(s)
-        :param cleanup:     Removes non-BIDS-compliant characters from the retrieved dynamic value if True
+        :param cleanup:     Sanitizes non-BIDS-compliant characters from the retrieved dynamic value if True
         :param runtime:     Replaces dynamic values if True
         :return:            Updated value
         """
@@ -1751,7 +1752,7 @@ def get_bidsname(subid: str, sesid: str, run: dict, validkeys: bool, runtime: bo
     :param run:         The run mapping with the BIDS key-value pairs
     :param validkeys:   Removes non-BIDS-compliant bids-keys if True
     :param runtime:     Replaces dynamic bidsvalues if True
-    :param cleanup:     Removes non-BIDS-compliant characters if True
+    :param cleanup:     Sanitizes non-BIDS-compliant characters from the filename if True
     :return:            The composed BIDS file-name (without file-extension)
     """
 
@@ -1951,37 +1952,68 @@ def increment_runindex(outfolder: Path, bidsname: str, run: dict, scans_table: D
     return f"{bidsname}.{suffixes}" if suffixes else bidsname
 
 
-def copymetadata(metasource: Path, metatarget: Path, extensions: list) -> dict:
+def poolmetadata(sourcemeta: Path, targetmeta: Path, usermeta: dict, extensions: list, datasource: DataSource) -> dict:
     """
-    Copies over or, in case of json-files, returns the content of 'metasource' data files
+    Load the metadata from the target (json sidecar), then add metadata from the source (json sidecar) and finally add
+    the user metadata (meta table). Source metadata other than json sidecars are copied over to the target folder
 
-    NB: In future versions this function could also support returning the content of e.g. csv- or Excel-files
+    NB: In future versions this function could also support more source metadata formats, e.g. yaml, csv- or Excel-files
 
-    :param metasource:  The filepath of the source-data file with associated/equally named meta-data files
-    :param metatarget:  The filepath of the source-data file to with the (non-json) meta-data files are copied over
-    :param extensions:  A list of file extensions of the meta-data files
-    :return:            The meta-data of the json-file
+    :param sourcemeta:  The filepath of the source data file with associated/equally named meta-data files (name may include wildcards)
+    :param targetmeta:  The filepath of the target data file with meta-data
+    :param usermeta:    A user metadata dict, e.g. the meta table from a run-item
+    :param extensions:  A list of file extensions of the source metadata files, e.g. as specified in bidsmap['Options']['plugins']['plugin']['meta']
+    :param datasource:  The data source from which dynamic values are read
+    :return:            The combined target + source + user metadata
     """
 
-    metadict = {}
+    metapool = {}
+
+    # Add the target metadata to the metadict
+    if targetmeta.is_file():
+        with targetmeta.open('r') as json_fid:
+            metapool = json.load(json_fid)
+
+    # Add the source metadata to the metadict or copy it over
     for ext in extensions:
-        metasource = metasource.with_suffix('').with_suffix(ext)
-        metatarget = metatarget.with_suffix('').with_suffix(ext)
-        if metasource.is_file():
-            LOGGER.info(f"Copying source data from: '{metasource}''")
+        for sourcefile in sourcemeta.parent.glob(sourcemeta.with_suffix('').with_suffix(ext).name):
+            LOGGER.info(f"Copying source data from: '{sourcefile}''")
+
+            # Put the metadata in metadict
             if ext == '.json':
-                with metasource.open('r') as json_fid:
-                    metadict = json.load(json_fid)
-                if not isinstance(metadict, dict):
-                    LOGGER.error(f"Skipping unexpectedly formatted meta-data in: {metasource}")
-                    metadict = {}
+                with sourcefile.open('r') as json_fid:
+                    metadata = json.load(json_fid)
+                if not isinstance(metadata, dict):
+                    LOGGER.error(f"Skipping unexpectedly formatted meta-data in: {sourcefile}")
+                    continue
+                for metakey, metaval in metadata.items():
+                    if metapool.get(metakey) and metapool.get(metakey) != metaval:
+                        LOGGER.info(f"Overruling {metakey} values in {targetmeta}: {metapool[metakey]} -> {metaval}")
+                    else:
+                        LOGGER.verbose(f"Adding '{metakey}: {metaval}' to: {targetmeta}")
+                    metapool[metakey] = metaval if metaval else None
+
+            # Or just copy over the metadata file
             else:
-                if metatarget.is_file():
-                    LOGGER.warning(f"Deleting unexpected existing data-file: {metatarget}")
-                    metatarget.unlink()
-                shutil.copy2(metasource, metatarget)
+                targetfile = targetmeta.parent/sourcefile.name
+                if not targetfile.is_file():
+                    shutil.copy2(sourcefile, targetfile)
+
+    # Add all the metadata to the metadict. NB: the dynamic `IntendedFor` value is handled separately later
+    for metakey, metaval in usermeta.items():
+        if metakey != 'IntendedFor':
+            metaval = datasource.dynamicvalue(metaval, cleanup=False, runtime=True)
+            try:
+                metaval = ast.literal_eval(str(metaval))  # E.g. convert stringified list or int back to list or int
+            except (ValueError, SyntaxError):
+                pass
+        if metapool.get(metakey) and metapool.get(metakey) != metaval:
+            LOGGER.info(f"Overruling {metakey} values in {targetmeta}: {metapool[metakey]} -> {metaval}")
+        else:
+            LOGGER.verbose(f"Adding '{metakey}: {metaval}' to: {targetmeta}")
+        metapool[metakey] = metaval if metaval else None
 
-    return metadict
+    return metapool
 
 
 def addparticipant(participants_tsv: Path, subid: str='', sesid: str='', data: dict=None) -> pd.DataFrame:

bidscoin/plugins/dcm2niix2bids.py

@@ -8,7 +8,6 @@
 import dateutil.parser
 import pandas as pd
 import json
-import ast
 import shutil
 from bids_validator import BIDSValidator
 from typing import Union
@@ -424,15 +423,11 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> None:
                 for oldfile in outfolder.glob(dcm2niixfile.with_suffix('').stem + '.*'):
                     oldfile.replace(newjsonfile.with_suffix(''.join(oldfile.suffixes)))
 
-        # Copy over the source meta-data
-        metadata = bids.copymetadata(sourcefile, outfolder/bidsname, options.get('meta', []))
-
         # Loop over all the newly produced json sidecar-files and adapt the data (NB: assumes every NIfTI-file comes with a json-file)
         for jsonfile in sorted(set(jsonfiles)):
 
-            # Load the json meta-data
-            with jsonfile.open('r') as json_fid:
-                jsondata = json.load(json_fid)
+            # Load / copy over the source meta-data
+            metadata = bids.poolmetadata(sourcefile, jsonfile, run['meta'], options['meta'], datasource)
 
             # Remove the bval/bvec files of sbref- and inv-images (produced by dcm2niix but not allowed by the BIDS specifications)
             if (datasource.datatype=='dwi' and suffix=='sbref') or (datasource.datatype=='fmap' and suffix=='epi'):
@@ -442,34 +437,17 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> None:
                         bdata = pd.read_csv(bfile, header=None)
                         if bdata.any(axis=None):
                             LOGGER.warning(f"Storing unexpected non-zero values from {bfile} -> {jsonfile}")
-                            jsondata[ext[1:]] = bdata.values.tolist()
+                            metadata[ext[1:]] = bdata.values.tolist()
                         LOGGER.verbose(f"Removing BIDS-invalid file: {bfile}")
                         bfile.unlink()
 
-            # Add all the source meta data to the meta-data
-            for metakey, metaval in metadata.items():
-                if jsondata.get(metakey) and jsondata.get(metakey) == metaval:
-                    LOGGER.warning(f"Overruling {metakey} values in {jsonfile}: {jsondata[metakey]} -> {metaval}")
-                jsondata[metakey] = metaval if metaval else None
-
-            # Add all the run meta data to the meta-data. NB: the dynamic `IntendedFor` value is handled separately later
-            for metakey, metaval in run['meta'].items():
-                if metakey != 'IntendedFor':
-                    metaval = datasource.dynamicvalue(metaval, cleanup=False, runtime=True)
-                    try: metaval = ast.literal_eval(str(metaval))            # E.g. convert stringified list or int back to list or int
-                    except (ValueError, SyntaxError): pass
-                    LOGGER.verbose(f"Adding '{metakey}: {metaval}' to: {jsonfile}")
-                if jsondata.get(metakey) and jsondata.get(metakey) == metaval:
-                    LOGGER.warning(f"Overruling {metakey} values in {jsonfile}: {jsondata[metakey]} -> {metaval}")
-                jsondata[metakey] = metaval if metaval else None
-
             # Remove unused (but added from the template) B0FieldIdentifiers/Sources
-            if not jsondata.get('B0FieldSource'):     jsondata.pop('B0FieldSource', None)
-            if not jsondata.get('B0FieldIdentifier'): jsondata.pop('B0FieldIdentifier', None)
+            if not metadata.get('B0FieldSource'):     metadata.pop('B0FieldSource', None)
+            if not metadata.get('B0FieldIdentifier'): metadata.pop('B0FieldIdentifier', None)
 
             # Save the meta-data to the json sidecar-file
             with jsonfile.open('w') as json_fid:
-                json.dump(jsondata, json_fid, indent=4)
+                json.dump(metadata, json_fid, indent=4)
 
             # Parse the acquisition time from the source header or else from the json file (NB: assuming the source file represents the first acquisition)
             outputfile = [file for file in jsonfile.parent.glob(jsonfile.stem + '.*') if file.suffix in ('.nii','.gz')]     # Find the corresponding NIfTI/tsv.gz file (there should be only one, let's not make assumptions about the .gz extension)
@@ -482,7 +460,7 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> None:
                 elif dataformat == 'PAR':
                     acq_time = datasource.attributes('exam_date')
                 if not acq_time or acq_time == 'T':
-                    acq_time = f"1925-01-01T{jsondata.get('AcquisitionTime','')}"
+                    acq_time = f"1925-01-01T{metadata.get('AcquisitionTime','')}"
                 try:
                     acq_time = dateutil.parser.parse(acq_time)
                     if options.get('anon','y') in ('y','yes'):

bidscoin/plugins/nibabel2bids.py

@@ -5,7 +5,6 @@
 import logging
 import dateutil.parser
 import json
-import ast
 import shutil
 import pandas as pd
 import nibabel as nib
@@ -169,7 +168,7 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> None:
 
     # Get started
     options     = bidsmap['Options']['plugins']['nibabel2bids']
-    ext         = options.get('ext', OPTIONS['ext'])
+    ext         = options.get('ext', '')
     meta        = options.get('meta', [])
     sourcefiles = [file for file in session.rglob('*') if is_sourcefile(file)]
     if not sourcefiles:
@@ -227,31 +226,20 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> None:
         # Save the sourcefile as a BIDS NIfTI file
         nib.save(nib.load(sourcefile), bidsfile)
 
-        # Copy over the source meta-data
-        jsonfile = bidsfile.with_suffix('').with_suffix('.json')
-        jsondata = bids.copymetadata(sourcefile, bidsfile, meta)
-
-        # Add all the metadata to the meta-data. NB: the dynamic `IntendedFor` value is handled separately later
-        for metakey, metaval in run['meta'].items():
-            if metakey != 'IntendedFor':
-                metaval = datasource.dynamicvalue(metaval, cleanup=False, runtime=True)
-                try: metaval = ast.literal_eval(str(metaval))            # E.g. convert stringified list or int back to list or int
-                except (ValueError, SyntaxError): pass
-                LOGGER.verbose(f"Adding '{metakey}: {metaval}' to: {jsonfile}")
-            if jsondata.get(metakey) and jsondata.get(metakey)==metaval:
-                LOGGER.warning(f"Overruling {metakey} values in {jsonfile}: {jsondata[metakey]} -> {metaval}")
-            jsondata[metakey] = metaval if metaval else None
+        # Load / copy over the source meta-data
+        sidecar  = bidsfile.with_suffix('').with_suffix('.json')
+        metadata = bids.poolmetadata(sourcefile, sidecar, run['meta'], meta, datasource)
 
         # Remove unused (but added from the template) B0FieldIdentifiers/Sources
-        if not jsondata.get('B0FieldSource'):     jsondata.pop('B0FieldSource', None)
-        if not jsondata.get('B0FieldIdentifier'): jsondata.pop('B0FieldIdentifier', None)
+        if not metadata.get('B0FieldSource'):     metadata.pop('B0FieldSource',     None)
+        if not metadata.get('B0FieldIdentifier'): metadata.pop('B0FieldIdentifier', None)
 
         # Save the meta-data to the json sidecar-file
-        with jsonfile.open('w') as json_fid:
-            json.dump(jsondata, json_fid, indent=4)
+        with sidecar.open('w') as json_fid:
+            json.dump(metadata, json_fid, indent=4)
 
         # Add an entry to the scans_table (we typically don't have useful data to put there)
-        acq_time = dateutil.parser.parse(f"1925-01-01T{jsondata.get('AcquisitionTime', '')}")
+        acq_time = dateutil.parser.parse(f"1925-01-01T{metadata.get('AcquisitionTime', '')}")
         scans_table.loc[bidsfile.relative_to(bidsses).as_posix(), 'acq_time'] = acq_time.isoformat()
 
     # Write the scans_table to disk

bidscoin/plugins/pet2bids.py

@@ -12,6 +12,7 @@
 import shutil
 import subprocess
 import pandas as pd
+import json
 from typing import Union
 from pathlib import Path
 from functools import lru_cache
@@ -271,7 +272,7 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> None:
 
             # Convert the source-files in the run folder to nifti's in the BIDS-folder
             else:
-                command = f'{options["command"]} "{source}" -d {outfolder/Path(bidsname).with_suffix(".nii.gz")}'
+                command = f'{options["command"]} "{source}" -d {(outfolder/bidsname).with_suffix(".nii.gz")}'
                 # pass in data added via bidseditor/bidsmap
                 if len(run.get('meta', {})) > 0:
                     command += ' --kwargs '
@@ -281,6 +282,12 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> None:
                 if bcoin.run_command(command):
                     if not list(outfolder.glob(f"{bidsname}.*nii*")): continue
 
+            # Load / copy over the source meta-data
+            sidecar  = (outfolder/bidsname).with_suffix('.json')
+            metadata = bids.poolmetadata(sourcefile, sidecar, run['meta'], options['meta'], datasource)
+            with sidecar.open('w') as json_fid:
+                json.dump(metadata, json_fid, indent=4)
+
     # Collect personal data from a source header and store it in the participants.tsv file
     if dataformat == 'DICOM':
         personals = {}