From 514162d1843042d5b3d39f57f715e0dab870043f Mon Sep 17 00:00:00 2001 From: Daniel Nilsson Date: Fri, 29 Nov 2024 15:17:40 +0100 Subject: [PATCH 01/17] Fix #261 - pkg_resources migration --- chanjo/__init__.py | 5 +++-- chanjo/cli/base.py | 15 ++++++++------- chanjo/init/demo.py | 39 +++++++++++++++++++++++---------------- requirements.txt | 1 + 4 files changed, 35 insertions(+), 25 deletions(-) diff --git a/chanjo/__init__.py b/chanjo/__init__.py index 75e77d3f..d2c41e8e 100644 --- a/chanjo/__init__.py +++ b/chanjo/__init__.py @@ -8,7 +8,8 @@ :licence: MIT, see LICENCE for more details """ import logging -from pkg_resources import get_distribution +from importlib.metadata import version + __banner__ = r""" ______ ________ @@ -23,7 +24,7 @@ __summary__ = 'coverage analysis tool for clinical sequencing' __uri__ = 'http://www.chanjo.co/' -__version__ = get_distribution(__title__).version +__version__ = version(__title__) __codename__ = 'Optimistic Otter' __author__ = 'Robin Andeer' diff --git a/chanjo/cli/base.py b/chanjo/cli/base.py index 103d5e54..f905868b 100644 --- a/chanjo/cli/base.py +++ b/chanjo/cli/base.py @@ -6,7 +6,7 @@ """ import logging import os -import pkg_resources +from importlib.metadata import entry_points import click import coloredlogs @@ -22,8 +22,9 @@ class EntryPointsCLI(click.MultiCommand): def _iter_commands(self): """Iterate over all sub-commands as defined by the entry point.""" - return {entry_point.name: entry_point for entry_point in - pkg_resources.iter_entry_points('chanjo.subcommands.4')} + # Get all entry points for the specified group + eps = entry_points(group='chanjo.subcommands.4') + return {ep.name: ep for ep in eps} def list_commands(self, ctx): """List the available commands.""" @@ -34,7 +35,7 @@ def get_command(self, ctx, name): """Load one of the available commands.""" commands = self._iter_commands() if name not in commands: - click.echo("no such command: {}".format(name)) + click.echo(f"no such command: {name}") ctx.abort() return commands[name].load() @@ -51,9 +52,9 @@ def root(context, config, database, log_level, log_file): """Clinical sequencing coverage analysis tool.""" logout = log_file or click.get_text_stream('stderr') coloredlogs.install(level=log_level, stream=logout) - LOG.debug("version {0}".format(__version__)) + LOG.debug("version %s", __version__) - # avoid setting global defaults in Click options, do it below when + # Load configuration from the provided file if os.path.exists(config): with open(config) as conf_handle: context.obj = yaml.safe_load(conf_handle) @@ -61,5 +62,5 @@ def root(context, config, database, log_level, log_file): context.obj = {} context.obj['database'] = (database or context.obj.get('database')) - # update the context with new defaults from the config file + # Update the context with new defaults from the config file context.default_map = context.obj diff --git a/chanjo/init/demo.py b/chanjo/init/demo.py index 166ce781..8ace8b40 100644 --- a/chanjo/init/demo.py +++ b/chanjo/init/demo.py @@ -1,37 +1,44 @@ # -*- coding: utf-8 -*- from errno import EEXIST import logging -from pkg_resources import resource_filename, resource_listdir - -from path import Path +import shutil +from pathlib import Path +from importlib.resources import files DEMO_BED_NAME = 'hgnc.min.bed' log = logging.getLogger(__name__) - def setup_demo(location, force=False): """Copy demo files to a directory. - \b - LOCATION: directory to add demofiles to (default: ./chanjo-demo) + LOCATION: directory to add demo files to (default: ./chanjo-demo) """ target_dir = Path(location) pkg_dir = __name__.rpartition('.')[0] - demo_dir = Path(resource_filename(pkg_dir, 'demo-files')) - # make sure we don't overwrite exiting files - for demo_file in resource_listdir(pkg_dir, 'demo-files'): - target_file_path = target_dir.joinpath(demo_file) + # Get the demo-files directory path using importlib.resources + demo_dir = files(pkg_dir) / 'demo-files' + + if not demo_dir.is_dir(): + log.error("Demo files directory does not exist") + raise FileNotFoundError(f"'demo-files' directory not found in package {pkg_dir}") + + # Check for existing files and avoid overwriting unless `force` is True + for demo_file in demo_dir.iterdir(): + target_file_path = target_dir / demo_file.name if not force and target_file_path.exists(): log.error("%s exists, pick a different location", target_file_path) raise OSError(EEXIST, 'file already exists', target_file_path) try: - # we can copy the directory(tree) - demo_dir.copytree(target_dir) + # Copy the directory tree + shutil.copytree(demo_dir, target_dir) + except FileExistsError: + log.warning('Location must be a non-existing directory') + raise except OSError as error: - log.warn('location must be a non-existing directory') - raise error + log.error('An error occurred during file copying: %s', error) + raise - # inform the user - log.info("successfully copied demo files to %s", target_dir) + # Inform the user + log.info("Successfully copied demo files to %s", target_dir) diff --git a/requirements.txt b/requirements.txt index bd3784aa..e676bf30 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ path.py toolz pyyaml importlib_metadata +importlib_resources pymysql sqlalchemy>=2.0 sqlservice>=3.0 From f4ab437877a42ed41e947a13eccc7d37d9cdcb37 Mon Sep 17 00:00:00 2001 From: Daniel Nilsson Date: Fri, 29 Nov 2024 15:41:52 +0100 Subject: [PATCH 02/17] first try the backport versions --- chanjo/__init__.py | 5 ++++- chanjo/cli/base.py | 5 ++++- chanjo/init/demo.py | 7 ++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/chanjo/__init__.py b/chanjo/__init__.py index d2c41e8e..7901d0ea 100644 --- a/chanjo/__init__.py +++ b/chanjo/__init__.py @@ -8,7 +8,10 @@ :licence: MIT, see LICENCE for more details """ import logging -from importlib.metadata import version +try: + from importlib.metadata import version +except ImportError: # Backport support for importlib metadata on Python 3.7 + from importlib_metadata import version __banner__ = r""" diff --git a/chanjo/cli/base.py b/chanjo/cli/base.py index f905868b..a42ffbf9 100644 --- a/chanjo/cli/base.py +++ b/chanjo/cli/base.py @@ -6,7 +6,10 @@ """ import logging import os -from importlib.metadata import entry_points +try: + from importlib.metadata import entry_points +except ImportError: # Backport support for importlib metadata on Python 3.7 + from importlib_metadata import entry_points import click import coloredlogs diff --git a/chanjo/init/demo.py b/chanjo/init/demo.py index 8ace8b40..4f88ee4c 100644 --- a/chanjo/init/demo.py +++ b/chanjo/init/demo.py @@ -3,7 +3,12 @@ import logging import shutil from pathlib import Path -from importlib.resources import files + +try: + from importlib.resources import files +except ImportError: # Backport support for importlib metadata on Python 3.7 + from importlib_resources import files + DEMO_BED_NAME = 'hgnc.min.bed' log = logging.getLogger(__name__) From 2bbd6cf3345c9b5702e39dc800d1db3009dbc845 Mon Sep 17 00:00:00 2001 From: Daniel Nilsson Date: Mon, 2 Dec 2024 09:13:27 +0100 Subject: [PATCH 03/17] make that a pre 3.10 fallback --- chanjo/cli/base.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/chanjo/cli/base.py b/chanjo/cli/base.py index a42ffbf9..40d73764 100644 --- a/chanjo/cli/base.py +++ b/chanjo/cli/base.py @@ -19,6 +19,7 @@ LOG = logging.getLogger(__name__) +COMMAND_GROUP_KEY = 'chanjo.subcommands.4' class EntryPointsCLI(click.MultiCommand): """Add sub-commands dynamically to a CLI via entry points.""" @@ -26,7 +27,11 @@ class EntryPointsCLI(click.MultiCommand): def _iter_commands(self): """Iterate over all sub-commands as defined by the entry point.""" # Get all entry points for the specified group - eps = entry_points(group='chanjo.subcommands.4') + if hasattr(entry_points(), 'select'): + # Python >3.10, importlib + eps = entry_points(group=COMMAND_GROUP_KEY) + else: + eps = entry_points().get(COMMAND_GROUP_KEY, []) return {ep.name: ep for ep in eps} def list_commands(self, ctx): From 68a9b7f5f5b83bf958451af7795c0a135b033a9c Mon Sep 17 00:00:00 2001 From: Daniel Nilsson Date: Mon, 2 Dec 2024 09:54:36 +0100 Subject: [PATCH 04/17] workaround for pathlib instead of os.path --- chanjo/store/api.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/chanjo/store/api.py b/chanjo/store/api.py index 1b701575..22f4eb62 100644 --- a/chanjo/store/api.py +++ b/chanjo/store/api.py @@ -3,6 +3,8 @@ import logging import os +from pathlib import Path + from sqlservice import Database from chanjo.calculate import CalculateMixin from .models import BASE @@ -11,6 +13,11 @@ LOG = logging.getLogger(__name__) +def get_absolute_path(db_uri): + """Get the absolute path of the database URI.""" + # Use Path for handling paths + db_path = Path(db_uri).expanduser().resolve() + return db_path class ChanjoDB(Database, CalculateMixin, DeleteMixin, FetchMixin): """SQLAlchemy-based database object. @@ -62,7 +69,7 @@ def connect(self, db_uri, debug=False): config['SQLALCHEMY_POOL_RECYCLE'] = 3600 elif '://' not in db_uri: # expect only a path to a sqlite database - db_path = os.path.abspath(os.path.expanduser(db_uri)) + db_path = get_absolute_path(db_uri) db_uri = "sqlite:///{}".format(db_path) config['SQLALCHEMY_DATABASE_URI'] = db_uri From 434bbd7c3fdde784770d09dd0b27dc12cc0a7a5f Mon Sep 17 00:00:00 2001 From: Daniel Nilsson Date: Mon, 2 Dec 2024 10:43:54 +0100 Subject: [PATCH 05/17] one more abspath --- chanjo/cli/init.py | 4 ++-- chanjo/cli/load.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/chanjo/cli/init.py b/chanjo/cli/init.py index ea9b15ca..a40104fb 100644 --- a/chanjo/cli/init.py +++ b/chanjo/cli/init.py @@ -4,7 +4,7 @@ import logging import click -from path import Path +from pathlib import Path import yaml from chanjo.store.api import ChanjoDB @@ -27,7 +27,7 @@ def init(context, force, demo, auto, root_dir): LOG.info("setting up chanjo under: %s", root_path) db_uri = context.obj.get('database') - db_uri = db_uri or "sqlite:///{}".format(root_path.joinpath(DB_NAME).abspath()) + db_uri = db_uri or "sqlite:///{}".format(root_path.joinpath(DB_NAME).resolve()) # test setup of sambamba sambamba_bin = find_executable('sambamba') diff --git a/chanjo/cli/load.py b/chanjo/cli/load.py index 3ad89b68..787951d0 100644 --- a/chanjo/cli/load.py +++ b/chanjo/cli/load.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- -import os.path import logging import sys +from pathlib import Path + import click from sqlalchemy.exc import IntegrityError @@ -12,7 +13,6 @@ LOG = logging.getLogger(__name__) - def validate_stdin(context, param, value): """Validate piped input contains some data. @@ -39,7 +39,7 @@ def validate_stdin(context, param, value): def load(context, sample, group, name, group_name, threshold, bed_stream): """Load Sambamba output into the database for a sample.""" chanjo_db = ChanjoDB(uri=context.obj['database']) - source = os.path.abspath(bed_stream.name) + source = Path(bed_stream.name).resolve() result = load_transcripts(bed_stream, sample_id=sample, group_id=group, source=source, threshold=threshold) From 2c6e7975e5023e0e7292aa4d0bc17e61326427a9 Mon Sep 17 00:00:00 2001 From: Daniel Nilsson Date: Mon, 2 Dec 2024 16:28:04 +0100 Subject: [PATCH 06/17] mkdirs as well --- chanjo/cli/init.py | 2 +- chanjo/init/bootstrap.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/chanjo/cli/init.py b/chanjo/cli/init.py index a40104fb..60ce5be4 100644 --- a/chanjo/cli/init.py +++ b/chanjo/cli/init.py @@ -53,7 +53,7 @@ def init(context, force, demo, auto, root_dir): is_bootstrapped = True # setup config file - root_path.makedirs_p() + root_path.mkdir(parents=True, exist_ok=True) conf_path = root_path.joinpath('chanjo.yaml') with open(conf_path, 'w') as conf_handle: data = {'database': db_uri} diff --git a/chanjo/init/bootstrap.py b/chanjo/init/bootstrap.py index 362c17f9..33026f99 100644 --- a/chanjo/init/bootstrap.py +++ b/chanjo/init/bootstrap.py @@ -3,12 +3,14 @@ import zipfile import sys +from pathlib import Path + if sys.version_info[0] >=3: from urllib.request import urlretrieve else: from urllib import urlretrieve -from path import Path + DB_NAME = 'chanjo.coverage.sqlite3' BED_NAME = 'hgnc.grch37p13.exons.bed' @@ -26,7 +28,7 @@ def pull(target_dir, force=False): # pragma: no cover """ logger.debug('ensure target directory exists') target_path = Path(target_dir) - target_path.makedirs_p() + target_path.mkdir(parents=True, exist_ok=True) bed_zip_path = target_path.joinpath("{}.zip".format(BED_NAME)) final_bed = target_path.joinpath(BED_NAME) From 9807bf108dde7a2866ce4d90451f66382c313587 Mon Sep 17 00:00:00 2001 From: Daniel Nilsson Date: Mon, 2 Dec 2024 16:34:50 +0100 Subject: [PATCH 07/17] one more.. --- chanjo/testutils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/chanjo/testutils.py b/chanjo/testutils.py index 883fe070..2c86c14e 100644 --- a/chanjo/testutils.py +++ b/chanjo/testutils.py @@ -1,5 +1,4 @@ -import os - +from pathlib import Path def fake_urlretrieve(url, target): open(target, 'a').close() @@ -11,5 +10,5 @@ def __init__(self, in_path, mode='r'): self.in_path = in_path def extractall(self, target_dir): - out_path = os.path.join(target_dir, self.in_path.replace('.zip', '')) + out_path = Path(target_dir.join(self.in_path.replace('.zip', ''))) open(out_path, 'a').close() From 1e7284dc56e01eea50177a7568ea66b6fbeec2c2 Mon Sep 17 00:00:00 2001 From: Daniel Nilsson Date: Mon, 2 Dec 2024 16:46:16 +0100 Subject: [PATCH 08/17] resolve a couple of more path obj to path strs --- chanjo/cli/init.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chanjo/cli/init.py b/chanjo/cli/init.py index 60ce5be4..76f7b612 100644 --- a/chanjo/cli/init.py +++ b/chanjo/cli/init.py @@ -38,14 +38,14 @@ def init(context, force, demo, auto, root_dir): if demo: LOG.info("copying demo files: %s", root_dir) - setup_demo(root_dir, force=force) + setup_demo(root_dir.resolve(), force=force) LOG.info("configure new chanjo database: %s", db_uri) chanjo_db = ChanjoDB(db_uri) chanjo_db.set_up() is_bootstrapped = True elif auto or click.confirm('Bootstrap HGNC transcript BED?'): - pull(root_dir, force=force) + pull(root_dir.resolve(), force=force) LOG.info("configure new chanjo database: %s", db_uri) chanjo_db = ChanjoDB(db_uri) From dcbea57a870b1749ecf6cb93a96cfb1946769790 Mon Sep 17 00:00:00 2001 From: Daniel Nilsson Date: Tue, 3 Dec 2024 14:08:59 +0100 Subject: [PATCH 09/17] fix bootstrap testing --- chanjo/cli/init.py | 4 ++-- chanjo/init/bootstrap.py | 2 +- chanjo/testutils.py | 7 ++++++- requirements.txt | 1 - tests/conftest.py | 10 +++++----- tests/init/test_init_bootstrap.py | 12 ++++++------ 6 files changed, 20 insertions(+), 16 deletions(-) diff --git a/chanjo/cli/init.py b/chanjo/cli/init.py index 76f7b612..60ce5be4 100644 --- a/chanjo/cli/init.py +++ b/chanjo/cli/init.py @@ -38,14 +38,14 @@ def init(context, force, demo, auto, root_dir): if demo: LOG.info("copying demo files: %s", root_dir) - setup_demo(root_dir.resolve(), force=force) + setup_demo(root_dir, force=force) LOG.info("configure new chanjo database: %s", db_uri) chanjo_db = ChanjoDB(db_uri) chanjo_db.set_up() is_bootstrapped = True elif auto or click.confirm('Bootstrap HGNC transcript BED?'): - pull(root_dir.resolve(), force=force) + pull(root_dir, force=force) LOG.info("configure new chanjo database: %s", db_uri) chanjo_db = ChanjoDB(db_uri) diff --git a/chanjo/init/bootstrap.py b/chanjo/init/bootstrap.py index 33026f99..a54baeba 100644 --- a/chanjo/init/bootstrap.py +++ b/chanjo/init/bootstrap.py @@ -42,6 +42,6 @@ def pull(target_dir, force=False): # pragma: no cover zip_ref.extractall(target_dir) logger.info('removing BED archive...') - bed_zip_path.remove_p() + bed_zip_path.unlink() else: logger.warn('file already exists, skipping: %s', final_bed) diff --git a/chanjo/testutils.py b/chanjo/testutils.py index 2c86c14e..29476471 100644 --- a/chanjo/testutils.py +++ b/chanjo/testutils.py @@ -1,5 +1,9 @@ from pathlib import Path +import logging + +LOG = logging.getLogger(__name__) + def fake_urlretrieve(url, target): open(target, 'a').close() @@ -10,5 +14,6 @@ def __init__(self, in_path, mode='r'): self.in_path = in_path def extractall(self, target_dir): - out_path = Path(target_dir.join(self.in_path.replace('.zip', ''))) + LOG.info(f"fake zip target dir : {target_dir} path {self.in_path}") + out_path = Path(target_dir).joinpath(self.in_path.name.replace('.zip', '')) open(out_path, 'a').close() diff --git a/requirements.txt b/requirements.txt index e676bf30..b0ff1e2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ coloredlogs click cryptography -path.py toolz pyyaml importlib_metadata diff --git a/tests/conftest.py b/tests/conftest.py index e856d41a..d85704aa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,7 @@ from chanjo.load.link import link_elements -@pytest.yield_fixture +@pytest.fixture def reset_path(): """Reset PATH environment variable temporarily.""" path_env = os.environ['PATH'] @@ -22,7 +22,7 @@ def reset_path(): os.environ['PATH'] = path_env -@pytest.yield_fixture(scope='function') +@pytest.fixture(scope='function') def chanjo_db(): _chanjo_db = ChanjoDB('sqlite://') _chanjo_db.set_up() @@ -30,7 +30,7 @@ def chanjo_db(): _chanjo_db.tear_down() -@pytest.yield_fixture(scope='function') +@pytest.fixture(scope='function') def existing_db(tmpdir): db_path = tmpdir.join('coverage.sqlite3') chanjo_db = ChanjoDB(str(db_path)) @@ -39,7 +39,7 @@ def existing_db(tmpdir): chanjo_db.tear_down() -@pytest.yield_fixture(scope='function') +@pytest.fixture(scope='function') def popexist_db(existing_db, exon_lines): with existing_db.begin(expire_on_commit=False) as session: result = link_elements(exon_lines) @@ -50,7 +50,7 @@ def popexist_db(existing_db, exon_lines): yield existing_db -@pytest.yield_fixture(scope='function') +@pytest.fixture(scope='function') def populated_db(chanjo_db, exon_lines): exon_lines = list(exon_lines) result = link_elements(exon_lines) diff --git a/tests/init/test_init_bootstrap.py b/tests/init/test_init_bootstrap.py index 70b185d7..b7995819 100644 --- a/tests/init/test_init_bootstrap.py +++ b/tests/init/test_init_bootstrap.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -import os +from pathlib import Path from mock import patch @@ -9,15 +9,15 @@ @patch('urllib.request.urlretrieve', fake_urlretrieve) @patch('zipfile.ZipFile', FakeZipFile) -def test_pull(tmpdir): +def test_pull(tmp_path): # GIVEN a target directory - target_dir = str(tmpdir) + target_dir = str(tmp_path) # WHEN downloading resources bootstrap.pull(target_dir) # THEN BED resource should be in place - out_bed = str(tmpdir.join(bootstrap.BED_NAME)) - assert os.path.exists(out_bed) - assert len(tmpdir.listdir()) == 1 + out_bed = tmp_path.joinpath(bootstrap.BED_NAME) + assert out_bed.exists() + assert len([tmp_path.iterdir()]) == 1 # GIVEN the resources already exists bootstrap.pull(target_dir) From 36c65f093c2fc7401f4cc54edb6b909aa096186f Mon Sep 17 00:00:00 2001 From: Daniel Nilsson Date: Tue, 3 Dec 2024 14:20:32 +0100 Subject: [PATCH 10/17] fix cli store test --- tests/cli/test_cli_store.py | 9 ++++----- tests/init/test_init_bootstrap.py | 4 +--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/cli/test_cli_store.py b/tests/cli/test_cli_store.py index c35f7232..7e717936 100644 --- a/tests/cli/test_cli_store.py +++ b/tests/cli/test_cli_store.py @@ -1,20 +1,19 @@ # -*- coding: utf-8 -*- - import json from chanjo.cli import root from chanjo.store.models import Sample, TranscriptStat -def test_setup(cli_runner, tmpdir): +def test_setup(cli_runner, tmp_path): # GIVEN an empty directory - assert tmpdir.listdir() == [] + assert not list(tmp_path.iterdir()) # WHEN setting up a new database db_name = "coverage.sqlite" - db_path = tmpdir.join(db_name) + db_path = tmp_path.joinpath(db_name) cli_runner.invoke(root, ["--database", str(db_path), "db", "setup"]) # THEN the file is created - assert tmpdir.listdir() == [db_path] + assert list(tmp_path.iterdir()) == [db_path] def test_setup_reset(cli_runner, popexist_db): diff --git a/tests/init/test_init_bootstrap.py b/tests/init/test_init_bootstrap.py index b7995819..84e9e804 100644 --- a/tests/init/test_init_bootstrap.py +++ b/tests/init/test_init_bootstrap.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from pathlib import Path - from mock import patch from chanjo.init import bootstrap @@ -17,7 +15,7 @@ def test_pull(tmp_path): # THEN BED resource should be in place out_bed = tmp_path.joinpath(bootstrap.BED_NAME) assert out_bed.exists() - assert len([tmp_path.iterdir()]) == 1 + assert len(list(tmp_path.iterdir())) == 1 # GIVEN the resources already exists bootstrap.pull(target_dir) From b7a1161c8b24f16585ef79453e6a2c2855e5896c Mon Sep 17 00:00:00 2001 From: Daniel Nilsson Date: Tue, 3 Dec 2024 14:26:03 +0100 Subject: [PATCH 11/17] fix test logging to file --- tests/cli/test_cli_base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/cli/test_cli_base.py b/tests/cli/test_cli_base.py index 60c3e3d1..2f0505f4 100644 --- a/tests/cli/test_cli_base.py +++ b/tests/cli/test_cli_base.py @@ -4,14 +4,14 @@ import yaml -def test_logging_to_file(tmpdir, invoke_cli): +def test_logging_to_file(tmp_path, invoke_cli): # GIVEN an empty directory - assert tmpdir.listdir() == [] + assert not list(tmp_path.iterdir()) # WHEN running the CLI to display some help for a subcommand - log_path = tmpdir.join('stderr.log') + log_path = tmp_path.joinpath('stderr.log') result = invoke_cli(['--log-file', str(log_path), 'db']) assert result.exit_code == 0 - assert tmpdir.listdir() == [log_path] + assert list(tmp_path.iterdir()) == [log_path] def test_list_commands(invoke_cli): From b234c5e482e10900eadea36636b575790273e780 Mon Sep 17 00:00:00 2001 From: Daniel Nilsson Date: Wed, 4 Dec 2024 11:11:19 +0100 Subject: [PATCH 12/17] fix a few more tests to use pathlib --- tests/cli/test_cli_init.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/cli/test_cli_init.py b/tests/cli/test_cli_init.py index 8fe6cf98..6f301446 100644 --- a/tests/cli/test_cli_init.py +++ b/tests/cli/test_cli_init.py @@ -7,40 +7,40 @@ from chanjo.testutils import FakeZipFile, fake_urlretrieve -def test_init_demo(tmpdir, invoke_cli): +def test_init_demo(tmp_path, invoke_cli): # GIVEN empty directory - assert tmpdir.listdir() == [] + assert not list(tmp_path.iterdir()) # WHEN setting up demo on CLI - target_dir = tmpdir.join('chanjo-demo') + target_dir = tmp_path.joinpath('chanjo-demo') target_path = str(target_dir) result = invoke_cli(['init', '--demo', target_path]) # THEN is should work and place 4 demo files + 1 sqlite db + 1 config assert result.exit_code == 0 - assert len(target_dir.listdir()) == (4 + 1 + 1) + assert len(list(target_dir.iterdir())) == (4 + 1 + 1) @patch('urllib.request.urlretrieve', fake_urlretrieve) @patch('zipfile.ZipFile', FakeZipFile) @patch('click.confirm', lambda param: True) -def test_init_bootstrap(tmpdir, invoke_cli): +def test_init_bootstrap(tmp_path, invoke_cli): # GIVEN empty dir - assert tmpdir.listdir() == [] - # WHEN boostrapping chanjo - result = invoke_cli(['init', str(tmpdir)]) + assert not list(tmp_path.iterdir()) + # WHEN bootstrapping chanjo + result = invoke_cli(['init', str(tmp_path)]) # THEN it should place 2 + 1 (BED, DB, config) files in the target dir assert result.exit_code == 0 - assert len(tmpdir.listdir()) == 3 + assert len(list(tmp_path.iterdir())) == 3 @patch('click.confirm', lambda param: False) @patch('click.prompt', lambda param: "{}/cov.sqlite3".format(gettempdir())) -def test_init_no_bootstrap(tmpdir, invoke_cli): +def test_init_no_bootstrap(tmp_path, invoke_cli): # GIVEN ... # WHEN opting out of bootstrap - result = invoke_cli(['init', str(tmpdir)]) + result = invoke_cli(['init', str(tmp_path)]) # THEN it should work with custom database URI assert result.exit_code == 0 - conf_path = tmpdir.join('chanjo.yaml') + conf_path = tmp_path.joinpath('chanjo.yaml') with open(str(conf_path), 'r') as handle: data = yaml.safe_load(handle) assert 'coverage.sqlite3' in data['database'] From 6c1b3a4364b1fe5551b73a5b7b054f0839454a07 Mon Sep 17 00:00:00 2001 From: Daniel Nilsson Date: Wed, 4 Dec 2024 11:34:25 +0100 Subject: [PATCH 13/17] shutil instead of find_executable --- chanjo/cli/init.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chanjo/cli/init.py b/chanjo/cli/init.py index 60ce5be4..5d8de303 100644 --- a/chanjo/cli/init.py +++ b/chanjo/cli/init.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- import codecs -from distutils.spawn import find_executable +from shutil import which import logging import click @@ -30,7 +30,7 @@ def init(context, force, demo, auto, root_dir): db_uri = db_uri or "sqlite:///{}".format(root_path.joinpath(DB_NAME).resolve()) # test setup of sambamba - sambamba_bin = find_executable('sambamba') + sambamba_bin = which('sambamba') if sambamba_bin is None: # pragma: no cover LOG.warning("'sambamba' command not found") else: From 7087675a7e48738d72cb364097a33081dd6906ce Mon Sep 17 00:00:00 2001 From: Daniel Nilsson Date: Wed, 4 Dec 2024 12:53:43 +0100 Subject: [PATCH 14/17] ah, str, not path --- chanjo/cli/load.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chanjo/cli/load.py b/chanjo/cli/load.py index 787951d0..a19a662c 100644 --- a/chanjo/cli/load.py +++ b/chanjo/cli/load.py @@ -39,7 +39,7 @@ def validate_stdin(context, param, value): def load(context, sample, group, name, group_name, threshold, bed_stream): """Load Sambamba output into the database for a sample.""" chanjo_db = ChanjoDB(uri=context.obj['database']) - source = Path(bed_stream.name).resolve() + source = str(Path(bed_stream.name).resolve()) result = load_transcripts(bed_stream, sample_id=sample, group_id=group, source=source, threshold=threshold) From eabbab91a2f95b0205a1ddad2cbbe1fc7a67489b Mon Sep 17 00:00:00 2001 From: Daniel Nilsson Date: Wed, 4 Dec 2024 14:31:34 +0100 Subject: [PATCH 15/17] Use more modern Dockerfile --- Dockerfile | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index cefa9884..2e2c273a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,13 @@ -FROM frolvlad/alpine-miniconda3 +FROM clinicalgenomics/python3.11-miniconda -LABEL base_image="frolvlad/alpine-miniconda3" +LABEL base_image="clinicalgenomics/python3.11-miniconda" LABEL about.home="https://github.com/Clinical-Genomics/chanjo" LABEL about.documentation="https://clinical-genomics.github.io/chanjo/" -LABEL about.tags="chanjo,bam,NGS,coverage,sambamba,alpine,python,python3.7" +LABEL about.tags="chanjo,bam,NGS,MPS,WES,WGS,coverage,sambamba,alpine,python" LABEL about.license="MIT License (MIT)" # Install Sambamba using conda -RUN conda update -n base -c defaults conda && conda install -c bioconda sambamba - -# Install required libs -RUN apk update \ - && apk --no-cache add bash python3 +RUN conda install -c bioconda sambamba WORKDIR /home/worker/app COPY . /home/worker/app @@ -23,6 +19,6 @@ RUN pip install -r requirements.txt RUN pip install -e . # Run commands as non-root user -RUN adduser -D worker +RUN adduser --disabled-login worker RUN chown worker:worker -R /home/worker USER worker From 3d2abf536a9c84c7d0ecf4ede7a913d5906590fd Mon Sep 17 00:00:00 2001 From: Daniel Nilsson Date: Wed, 4 Dec 2024 14:44:12 +0100 Subject: [PATCH 16/17] Update chanjo/testutils.py Co-authored-by: Chiara Rasi --- chanjo/testutils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/chanjo/testutils.py b/chanjo/testutils.py index 29476471..a797a302 100644 --- a/chanjo/testutils.py +++ b/chanjo/testutils.py @@ -14,6 +14,5 @@ def __init__(self, in_path, mode='r'): self.in_path = in_path def extractall(self, target_dir): - LOG.info(f"fake zip target dir : {target_dir} path {self.in_path}") out_path = Path(target_dir).joinpath(self.in_path.name.replace('.zip', '')) open(out_path, 'a').close() From 9f5b2da3772e3a7d32d9a6da159330d6c9fe5f92 Mon Sep 17 00:00:00 2001 From: Daniel Nilsson Date: Wed, 4 Dec 2024 14:44:56 +0100 Subject: [PATCH 17/17] main link --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d4cac918..933f2b35 100755 --- a/setup.py +++ b/setup.py @@ -68,7 +68,7 @@ def run_tests(self): license='MIT', # The project's main homepage - url='http://www.chanjo.co/', + url='https://clinical-genomics.github.io/chanjo/', packages=find_packages(exclude=('tests*', 'docs', 'examples')),