ENH: Add OTP processing option

.circleci/config.yml

@@ -2,16 +2,37 @@ version: 2
 jobs:
   build_docs:
     docker:
-      - image: circleci/python:3.7-buster
+      - image: cimg/base:stable-20.04
     steps:
       # Get our data and merge with upstream
       - checkout
-      - run: sudo apt-get install graphviz-dev graphviz
-      - run: python -m pip install --quiet --upgrade --user pip
-      - run: python -m pip install --quiet --upgrade --user numpy scipy https://github.com/mne-tools/mne-python/zipball/main numpydoc sphinx sphinx_fontawesome sphinx_bootstrap_theme pygraphviz
-      - run: python -c "import mne; mne.sys_info()"
-      - run: python setup.py develop --user
-      - run: cd doc && make html
+      - run:
+          name: Setup system
+          command: |
+            set -e
+            echo "set -e" >> $BASH_ENV
+            sudo apt update -qq
+            sudo apt install -qq graphviz-dev graphviz python3-venv python3.9-venv python3.9-dev
+            python3.9 -m venv ~/python_env
+            source ~/python_env/bin/activate
+            echo "source ~/python_env/bin/activate" >> $BASH_ENV
+            mkdir -p ~/.local/bin
+            ln -s ~/python_env/bin/python ~/.local/bin/python
+      - run:
+          name: Setup Python env
+          command: |
+            echo "Pip..."
+            pip install --quiet --upgrade pip
+            echo "Dependencies..."
+            pip install --quiet --upgrade numpy scipy https://github.com/mne-tools/mne-python/zipball/main numpydoc sphinx sphinx_fontawesome sphinx_bootstrap_theme
+            echo "PyGraphViz..."
+            pip install --upgrade pygraphviz
+            python -c "import mne; mne.sys_info()"
+            pip install -ve .
+      - run:
+          name: Build docs
+          command: |
+            make -C doc html
 
       - store_artifacts:
           path: doc/_build/html/
@@ -25,7 +46,7 @@ jobs:
 
   deploy:
     docker:
-      - image: circleci/python:3.6-stretch
+      - image: cimg/base:stable-20.04
     steps:
       - add_ssh_keys:
           fingerprints:

.github/workflows/pytest.yml

@@ -0,0 +1,41 @@
+name: 'pytest'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.number }}-${{ github.event.ref }}
+  cancel-in-progress: true
+on:
+  push:
+    branches:
+      - '*'
+  pull_request:
+    branches:
+      - '*'
+
+jobs:
+  job:
+    name: 'py3.10'
+    runs-on: ubuntu-20.04
+    defaults:
+      run:
+        shell: bash
+    env:
+      MNE_LOGGING_LEVEL: 'info'
+      MNE_STIM_CHANNEL: 'STI101'
+      OPENBLAS_NUM_THREADS: '1'
+      PYTHON_VERSION: '3.10'
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+        name: 'Setup python'
+      - shell: bash -el {0}
+        run: |
+          pip install --upgrade --only-binary=":all:" numpy scipy matplotlib
+          pip install --upgrade pyyaml patsy pytest pytest-cov codecov https://github.com/mne-tools/mne-python/archive/main.zip
+          pip install -ve .
+      - shell: bash -el {0}
+        run: pytest mnefun
+        name: 'Run tests'
+      - uses: codecov/codecov-action@v1
+        if: success()
+        name: 'Upload coverage to CodeCov'

.github/workflows/style.yml

@@ -0,0 +1,30 @@
+name: 'style'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.number }}-${{ github.event.ref }}
+  cancel-in-progress: true
+on:
+  push:
+    branches:
+      - '*'
+  pull_request:
+    branches:
+      - '*'
+
+jobs:
+  job:
+    name: 'flake8 + pydocstyle'
+    runs-on: ubuntu-20.04
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.10'
+        name: 'Setup python'
+      - shell: bash -el {0}
+        run: |
+          pip install --upgrade flake8 pydocstyle
+      - shell: bash -el {0}
+        run: make flake

doc/overview.rst

@@ -133,9 +133,19 @@ score : callable | None
 on_process : callable
     Called at each processing step.
 
-.. _do_sss:
+3. do_otp
+---------
+Run OTP processing.
+
+1. Run OTP locally using :func:`mne.preprocessing.oversampled_temporal_projection`.
+
+``preprocessing: otp``: OTP parameters
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+otp_dur : float | None
+    The window duration (in seconds) to use. Must be > or equal to tsss_dur if combining with tSSS processing.
 
-3. do_sss
+
+4. do_sss
 ---------
 
 .. warning:: Before running SSS, set ``params.mf_prebad[SUBJ]`` to a
@@ -281,12 +291,12 @@ cont_as_esss : bool
     Only supported when Python is used for SSS.
 
 
-4. do_ch_fix
+5. do_ch_fix
 ------------
 
 Fix EEG channel ordering, and also anonymize files.
 
-5. gen_ssp
+6. gen_ssp
 ----------
 
 .. warning:: Before running SSP, examine SSS'ed files and make
@@ -441,12 +451,12 @@ plot_drop_logs : bool
     If True, plot drop logs after preprocessing.
 
 
-6. apply_ssp
+7. apply_ssp
 ------------
 Apply SSP vectors and filtering to the files.
 
 
-7. write_epochs
+8. write_epochs
 ---------------
 Write epochs to disk.
 
@@ -537,7 +547,7 @@ allow_resample : bool
     matching sample rate. This is useful when recordings were errantly
     performed at different sample rates.
 
-8. gen_covs
+9. gen_covs
 -----------
 Generate covariances.
 
@@ -566,8 +576,8 @@ force_erm_cov_rank_full : bool
     is short and/or there are a lot of head movements.
 
 
-9. gen_fwd
-----------
+10. gen_fwd
+-----------
 .. warning:: Make SUBJ/trans/SUBJ-trans.fif using :ref:`mne:mne coreg`.
 
 Generate forward solutions (and source space if necessary).
@@ -591,7 +601,7 @@ fwd_mindist : float
     Minimum distance (mm) for sources in the brain from the skull in order
     for them to be included in the forward solution source space.
 
-10. gen_inv
+11. gen_inv
 -----------
 
 Generate inverses.
@@ -604,7 +614,7 @@ inv_runs : list of int
     Runs to use for each inverse.
 
 
-11. gen_report
+12. gen_report
 --------------
 
 Write :class:`mne.Report` HTML of results to disk.
@@ -677,8 +687,12 @@ keep_orig : bool
     Keep original files after anonymization.
 raw_fif_tag : str
     File tag for raw data, usually "_raw.fif".
+otp_fif_tag : str
+    File tag for OTP-processed files, usually "_raw_otp.fif".
 sss_fif_tag : str
     File tag for SSS-processed files, usually "_raw_sss.fif".
+otp_dir : str
+    Directory for OTP-processed files, usually "otp_fif".
 sss_dir : str
     Directory to use for SSS processed files, usually "sss_fif".
 pca_dir : str

mnefun/_flow.py

@@ -46,6 +46,7 @@ def _create_flowchart(fname):
         cov='./SUBJ/cov/\n*-cov.fif',
         lst='./SUBJ/lists/\nALL_*-eve.lst',
         raw='./SUBJ/raw_fif/\n*_raw.fif',
+        otp='./SUBJ/otp_fif/\n*_raw_otp.fif',
         sss='./SUBJ/sss_fif/\n*_raw_sss.fif',
         pca='./SUBJ/sss_pca_fif/\n*_fil55_raw_sss.fif',
         pro='./SUBJ/sss_pca_fif/\n*-proj.fif',
@@ -64,41 +65,42 @@ def _create_flowchart(fname):
         ('mri', 'src'),
         ('mri', 'tra', 'mne coreg'),
         ('acq', 'raw', '1. fetch_raw'),
-        ('raw', 'sco'),
-        ('sco', 'lst', '2. do_score'),
+        ('raw', 'otp', '2. do_otp'),
+        ('otp', 'sco'),
+        ('sco', 'lst', '3. do_score'),
         ('raw', 'mfb'),
-        ('raw', 'pbd'),
-        ('pbd', 'mfb', '3. do_sss'),
+        ('otp', 'pbd'),
+        ('pbd', 'mfb', '4. do_sss'),
         ('mfb', 'mfp'),
         ('pbd', 'mfp'),
         ('mfp', 'sss'),
         ('mfp', 'aan'),
-        ('raw', 'sss'),
+        ('otp', 'sss'),
         ('pbd', 'sss'),
         ('mfb', 'sss'),
         ('aan', 'sss'),
-        ('raw', 'can'),
+        ('otp', 'can'),
         ('can', 'sss'),
-        ('sss', 'sss', '4. do_ch_fix'),
+        ('sss', 'sss', '5. do_ch_fix'),
         ('bad', 'pro'),
         ('pex', 'pro'),
-        ('sss', 'pro', '5. gen_ssp'),
+        ('sss', 'pro', '6. gen_ssp'),
         ('sss', 'bad'),
         ('sss', 'pex'),
         ('bad', 'pca'),
         ('pex', 'pca'),
-        ('pro', 'pca', '6. apply_ssp'),
+        ('pro', 'pca', '7. apply_ssp'),
         ('sss', 'pca'),
-        ('pca', 'evo', '7. write_epochs'),
+        ('pca', 'evo', '8. write_epochs'),
         ('lst', 'evo'),
-        ('pca', 'cov', '8. gen_covs'),
+        ('pca', 'cov', '9. gen_covs'),
         ('bem', 'fwd'),
         ('pca', 'fwd'),
-        ('tra', 'fwd', '9. gen_fwd'),
+        ('tra', 'fwd', '10. gen_fwd'),
         ('src', 'fwd'),
-        ('fwd', 'inv', '10. gen_inv'),
+        ('fwd', 'inv', '11. gen_inv'),
         ('cov', 'inv'),
-        ('evo', 'htm', '11. gen_report'),
+        ('evo', 'htm', '12. gen_report'),
         ('cov', 'htm'),
         ('inv', 'htm'),
     )

mnefun/_mnefun.py

@@ -25,6 +25,7 @@
 from ._status import print_proc_status
 from ._paths import _get_config_file
 from ._utils import timestring
+from ._otp import run_otp
 
 
 # Class adapted from:
@@ -156,6 +157,7 @@ def __init__(self, tmin=None, tmax=None, t_adjust=0, bmin=-0.2, bmax=0.0,
         self.trans_dir = 'trans'
         self.bad_dir = 'bads'
         self.raw_dir = 'raw_fif'
+        self.otp_dir = 'otp_fif'
         self.sss_dir = 'sss_fif'
         self.pca_dir = 'sss_pca_fif'
 
@@ -167,6 +169,7 @@ def __init__(self, tmin=None, tmax=None, t_adjust=0, bmin=-0.2, bmax=0.0,
         self.inv_erm_tag = '-erm'
         self.eq_tag = 'eq'
         self.sss_fif_tag = '_raw_sss.fif'
+        self.otp_fif_tag = '_raw_otp.fif'
         self.bad_tag = '_post-sss.txt'
         self.keep_orig = False
         # This is used by fix_eeg_channels to fix original files
@@ -178,6 +181,7 @@ def __init__(self, tmin=None, tmax=None, t_adjust=0, bmin=-0.2, bmax=0.0,
         self.hp_type = 'maxfilter'
         self.mf_args = ''
         self.tsss_dur = 60.
+        self.otp_dur = None
         self.trans_to = 'median'  # where to transform head positions to
         self.sss_format = 'float'  # output type for MaxFilter
         self.movecomp = 'inter'
@@ -365,9 +369,9 @@ def _set_static(p):
                           ' processing will fail.' % (key,))
 
 
-def do_processing(p, fetch_raw=False, do_score=False, push_raw=False,
-                  do_sss=False, fetch_sss=False, do_ch_fix=False,
-                  gen_ssp=False, apply_ssp=False,
+def do_processing(p, *, fetch_raw=False, do_score=False, push_raw=False,
+                  do_otp=False, do_sss=False, fetch_sss=False,
+                  do_ch_fix=False, gen_ssp=False, apply_ssp=False,
                   write_epochs=False, gen_covs=False, gen_fwd=False,
                   gen_inv=False, gen_report=False, print_status=True):
     """Do M/EEG data processing.
@@ -382,6 +386,8 @@ def do_processing(p, fetch_raw=False, do_score=False, push_raw=False,
         Do scoring.
     push_raw : bool
         Push raw recording files to SSS workstation.
+    do_otp: bool
+        Apply OTP locally.
     do_sss : bool
         Run SSS remotely on SSS workstation.
     fetch_sss : bool
@@ -413,6 +419,7 @@ def do_processing(p, fetch_raw=False, do_score=False, push_raw=False,
     bools = [fetch_raw,
              do_score,
              push_raw,
+             do_otp,
              do_sss,
              fetch_sss,
              do_ch_fix,
@@ -428,6 +435,7 @@ def do_processing(p, fetch_raw=False, do_score=False, push_raw=False,
     texts = ['Pulling raw files from acquisition machine',
              'Scoring subjects',
              'Pushing raw files to remote workstation',
+             'Running OTP',
              'Running SSS using %s' % p.sss_type,
              'Pulling SSS files from remote workstation',
              'Fixing EEG order',
@@ -449,6 +457,7 @@ def score_fun(p, subjects, run_indices):
     funcs = [fetch_raw_files,
              score_fun,
              push_raw_files,
+             run_otp,
              run_sss,
              fetch_sss_files,
              fix_eeg_files,