From 9825d89ebb7b0616d100b70f09bf2962d055f3f3 Mon Sep 17 00:00:00 2001 From: jvfe Date: Fri, 21 Jun 2024 15:42:18 -0300 Subject: [PATCH 01/55] tests: Add test case for cross-organization subwf Strangely it is passing at the moment, it seems the install command returns a passing exit code regardless of installing all modules. - Addresses sanger-tol/nf-core-tools#2 --- tests/subworkflows/install.py | 9 +++++++++ tests/test_subworkflows.py | 1 + tests/utils.py | 1 + 3 files changed, 11 insertions(+) diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index dfe71686fb..477818b261 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -6,7 +6,9 @@ from nf_core.subworkflows.install import SubworkflowInstall from ..utils import ( + CROSS_ORGANIZATION_URL, GITLAB_BRANCH_TEST_BRANCH, + GITLAB_DEFAULT_BRANCH, GITLAB_REPO, GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH, @@ -79,6 +81,13 @@ def test_subworkflows_install_different_branch_fail(self): assert install_obj.install("bam_stats_samtools") is False +def test_subworkflows_install_across_organizations(self): + """Test installing a subworkflow with modules from different organizations""" + install_obj = SubworkflowInstall(self.pipeline_dir, remote_url=CROSS_ORGANIZATION_URL, branch=GITLAB_DEFAULT_BRANCH) + # The hic_bwamem2 subworkflow contains modules from different organizations + assert install_obj.install("hic_bwamem2") is True + + def test_subworkflows_install_tracking(self): """Test installing a subworkflow and finding the correct entries in installed_by section of modules.json""" self.subworkflow_install.install("bam_sort_stats_samtools") diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 0a9224002a..eeee62bda1 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -120,6 +120,7 @@ def tearDown(self): ) from .subworkflows.install import ( # type: ignore[misc] test_subworkflow_install_nopipeline, + test_subworkflows_install_across_organizations, test_subworkflows_install_alternate_remote, test_subworkflows_install_bam_sort_stats_samtools, test_subworkflows_install_bam_sort_stats_samtools_twice, diff --git a/tests/utils.py b/tests/utils.py index 89c1328818..34c899ccf3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -15,6 +15,7 @@ OLD_TRIMGALORE_SHA = "9b7a3bdefeaad5d42324aa7dd50f87bea1b04386" OLD_TRIMGALORE_BRANCH = "mimic-old-trimgalore" GITLAB_URL = "https://gitlab.com/nf-core/modules-test.git" +CROSS_ORGANIZATION_URL = "https://github.com/jvfe/test-subworkflow-remote.git" GITLAB_REPO = "nf-core-test" GITLAB_DEFAULT_BRANCH = "main" GITLAB_SUBWORKFLOWS_BRANCH = "subworkflows" From 568363a4b025d6bdfbce031a12e9898df3ca0137 Mon Sep 17 00:00:00 2001 From: jvfe Date: Fri, 28 Jun 2024 15:33:56 -0300 Subject: [PATCH 02/55] tests: Check module on module list instead --- tests/subworkflows/install.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index 477818b261..44856ad033 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -85,7 +85,13 @@ def test_subworkflows_install_across_organizations(self): """Test installing a subworkflow with modules from different organizations""" install_obj = SubworkflowInstall(self.pipeline_dir, remote_url=CROSS_ORGANIZATION_URL, branch=GITLAB_DEFAULT_BRANCH) # The hic_bwamem2 subworkflow contains modules from different organizations - assert install_obj.install("hic_bwamem2") is True + install_obj.install("hic_bwamem2") + # Verify that the installed_by entry was added correctly + modules_json = ModulesJson(self.pipeline_dir) + mod_json = modules_json.get_modules_json() + assert mod_json["repos"][CROSS_ORGANIZATION_URL]["modules"]["jvfe"]["samtools/merge"]["installed_by"] == [ + "hic_bwamem2" + ] def test_subworkflows_install_tracking(self): From 534da92b483587a4287ab9fa75331c57e06c87de Mon Sep 17 00:00:00 2001 From: jvfe Date: Mon, 1 Jul 2024 08:41:07 -0300 Subject: [PATCH 03/55] chore: Dummy commit to test action --- tests/subworkflows/install.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index 44856ad033..b654921674 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -89,6 +89,7 @@ def test_subworkflows_install_across_organizations(self): # Verify that the installed_by entry was added correctly modules_json = ModulesJson(self.pipeline_dir) mod_json = modules_json.get_modules_json() + # Dummy comment assert mod_json["repos"][CROSS_ORGANIZATION_URL]["modules"]["jvfe"]["samtools/merge"]["installed_by"] == [ "hic_bwamem2" ] From 286ab86263be1cb2d31c0093c58554f7c05c1e8f Mon Sep 17 00:00:00 2001 From: jvfe Date: Mon, 1 Jul 2024 08:45:13 -0300 Subject: [PATCH 04/55] Revert "chore: Dummy commit to test action" This reverts commit 534da92b483587a4287ab9fa75331c57e06c87de. --- tests/subworkflows/install.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index b654921674..44856ad033 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -89,7 +89,6 @@ def test_subworkflows_install_across_organizations(self): # Verify that the installed_by entry was added correctly modules_json = ModulesJson(self.pipeline_dir) mod_json = modules_json.get_modules_json() - # Dummy comment assert mod_json["repos"][CROSS_ORGANIZATION_URL]["modules"]["jvfe"]["samtools/merge"]["installed_by"] == [ "hic_bwamem2" ] From 423517ef1a268da1186f707d84d82de466d50df6 Mon Sep 17 00:00:00 2001 From: jvfe Date: Tue, 2 Jul 2024 18:46:48 -0300 Subject: [PATCH 05/55] tests: Change installed subworkflow --- tests/subworkflows/install.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index 44856ad033..f872b1f7fb 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -85,12 +85,12 @@ def test_subworkflows_install_across_organizations(self): """Test installing a subworkflow with modules from different organizations""" install_obj = SubworkflowInstall(self.pipeline_dir, remote_url=CROSS_ORGANIZATION_URL, branch=GITLAB_DEFAULT_BRANCH) # The hic_bwamem2 subworkflow contains modules from different organizations - install_obj.install("hic_bwamem2") + install_obj.install("get_genome_annotation") # Verify that the installed_by entry was added correctly modules_json = ModulesJson(self.pipeline_dir) mod_json = modules_json.get_modules_json() - assert mod_json["repos"][CROSS_ORGANIZATION_URL]["modules"]["jvfe"]["samtools/merge"]["installed_by"] == [ - "hic_bwamem2" + assert mod_json["repos"][CROSS_ORGANIZATION_URL]["modules"]["jvfe"]["prokka"]["installed_by"] == [ + "get_genome_annotation" ] From af816fd5a3a0ecf5e7317869eb5f3f9f86c5662c Mon Sep 17 00:00:00 2001 From: jvfe Date: Mon, 8 Jul 2024 15:45:34 -0300 Subject: [PATCH 06/55] feat: Read from meta.yml in get_components_to_install --- nf_core/components/components_utils.py | 10 ++++++---- nf_core/components/install.py | 7 ++++++- nf_core/modules/modules_json.py | 5 +++++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 01650a643d..1d433512ac 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -5,6 +5,7 @@ import questionary import rich.prompt +import yaml import nf_core.utils from nf_core.modules.modules_repo import ModulesRepo @@ -146,9 +147,10 @@ def get_components_to_install(subworkflow_dir: str) -> Tuple[List[str], List[str match = regex.match(line) if match and len(match.groups()) == 2: name, link = match.groups() - if link.startswith("../../../"): - name_split = name.lower().split("_") - modules.append("/".join(name_split)) - elif link.startswith("../"): + if link.startswith("../"): subworkflows.append(name.lower()) + with open(Path(subworkflow_dir, "meta.yml")) as fh: + meta = yaml.safe_load(fh) + components = meta.get("components") + modules.extend(components) return modules, subworkflows diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 6385ee4092..624bd072e3 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -14,7 +14,7 @@ prompt_component_version_sha, ) from nf_core.modules.modules_json import ModulesJson -from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME +from nf_core.modules.modules_repo import NF_CORE_MODULES_NAME, ModulesRepo log = logging.getLogger(__name__) @@ -53,6 +53,11 @@ def install(self, component, silent=False): # Check modules directory structure self.check_modules_structure() + if isinstance(component, dict): + component_name = list(component.keys())[0] + self.modules_repo = ModulesRepo(component[component_name]["git_remote"]) + component = component_name + # Verify that 'modules.json' is consistent with the installed modules and subworkflows modules_json = ModulesJson(self.dir) if not silent: diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 7d78268e92..dc47843601 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -1186,6 +1186,11 @@ def recreate_dependencies(self, repo, org, subworkflow): dep_mods, dep_subwfs = get_components_to_install(sw_path) for dep_mod in dep_mods: + if isinstance(dep_mod, dict): + component_name = list(dep_mod.keys())[0] + repo = dep_mod[component_name]["git_remote"] + org = dep_mod[component_name]["org_path"] + dep_mod = component_name installed_by = self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"] if installed_by == ["modules"]: self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"] = [] From d0e6031298f76b3e90b8f8e64bbca9892969b077 Mon Sep 17 00:00:00 2001 From: jvfe Date: Mon, 15 Jul 2024 09:52:07 -0300 Subject: [PATCH 07/55] refact: Change module return type to always be dict --- nf_core/components/components_utils.py | 18 ++++++++++-- nf_core/components/install.py | 39 +++++++++++++------------- nf_core/modules/modules_json.py | 8 +++--- 3 files changed, 39 insertions(+), 26 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 1d433512ac..0697fcd15f 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -133,7 +133,7 @@ def prompt_component_version_sha( return git_sha -def get_components_to_install(subworkflow_dir: str) -> Tuple[List[str], List[str]]: +def get_components_to_install(subworkflow_dir: str) -> Tuple[List[dict], List[str]]: """ Parse the subworkflow main.nf file to retrieve all imported modules and subworkflows. """ @@ -147,10 +147,22 @@ def get_components_to_install(subworkflow_dir: str) -> Tuple[List[str], List[str match = regex.match(line) if match and len(match.groups()) == 2: name, link = match.groups() - if link.startswith("../"): + if link.startswith("../") and not link.startswith("../../"): subworkflows.append(name.lower()) with open(Path(subworkflow_dir, "meta.yml")) as fh: meta = yaml.safe_load(fh) components = meta.get("components") - modules.extend(components) + component_list = [] + for component in components: + if isinstance(component, str): + comp_dict = {"name": component, "org_path": None, "git_remote": None} + else: + name = list(component.keys())[0] + comp_dict = { + "name": name, + "org_path": component[name]["org_path"], + "git_remote": component[name]["git_remote"], + } + component_list.append(comp_dict) + modules.extend(component_list) return modules, subworkflows diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 624bd072e3..d76a65bf45 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -53,10 +53,13 @@ def install(self, component, silent=False): # Check modules directory structure self.check_modules_structure() + repo_path = self.modules_repo.repo_path + remote_url = self.modules_repo.remote_url if isinstance(component, dict): - component_name = list(component.keys())[0] - self.modules_repo = ModulesRepo(component[component_name]["git_remote"]) - component = component_name + if component["git_remote"] is not None: + repo_path = component["org_path"] + remote_url = component["git_remote"] + component = component["name"] # Verify that 'modules.json' is consistent with the installed modules and subworkflows modules_json = ModulesJson(self.dir) @@ -70,61 +73,59 @@ def install(self, component, silent=False): return False # Verify SHA - if not self.modules_repo.verify_sha(self.prompt, self.sha): + if not ModulesRepo(remote_url).verify_sha(self.prompt, self.sha): return False # Check and verify component name - component = self.collect_and_verify_name(component, self.modules_repo) + component = self.collect_and_verify_name(component, ModulesRepo(remote_url)) if not component: return False # Get current version - current_version = modules_json.get_component_version( - self.component_type, component, self.modules_repo.remote_url, self.modules_repo.repo_path - ) + current_version = modules_json.get_component_version(self.component_type, component, remote_url, repo_path) # Set the install folder based on the repository name - install_folder = Path(self.dir, self.component_type, self.modules_repo.repo_path) + install_folder = Path(self.dir, self.component_type, repo_path) # Compute the component directory component_dir = Path(install_folder, component) # Check that the component is not already installed component_not_installed = self.check_component_installed( - component, current_version, component_dir, self.modules_repo, self.force, self.prompt, silent + component, current_version, component_dir, ModulesRepo(remote_url), self.force, self.prompt, silent ) if not component_not_installed: log.debug( f"{self.component_type[:-1].title()} is already installed and force is not set.\nAdding the new installation source {self.installed_by} for {self.component_type[:-1]} {component} to 'modules.json' without installing the {self.component_type}." ) modules_json.load() - modules_json.update(self.component_type, self.modules_repo, component, current_version, self.installed_by) + modules_json.update( + self.component_type, ModulesRepo(remote_url), component, current_version, self.installed_by + ) return False - version = self.get_version(component, self.sha, self.prompt, current_version, self.modules_repo) + version = self.get_version(component, self.sha, self.prompt, current_version, ModulesRepo(remote_url)) if not version: return False # Remove component if force is set and component is installed install_track = None if self.force: - log.debug(f"Removing installed version of '{self.modules_repo.repo_path}/{component}'") + log.debug(f"Removing installed version of '{repo_path}/{component}'") self.clear_component_dir(component, component_dir) - install_track = self.clean_modules_json(component, self.modules_repo, modules_json) + install_track = self.clean_modules_json(component, ModulesRepo(remote_url), modules_json) if not silent: log.info(f"{'Rei' if self.force else 'I'}nstalling '{component}'") - log.debug( - f"Installing {self.component_type} '{component}' at modules hash {version} from {self.modules_repo.remote_url}" - ) + log.debug(f"Installing {self.component_type} '{component}' at modules hash {version} from {remote_url}") # Download component files - if not self.install_component_files(component, version, self.modules_repo, install_folder): + if not self.install_component_files(component, version, ModulesRepo(remote_url), install_folder): return False # Update module.json with newly installed subworkflow modules_json.load() modules_json.update( - self.component_type, self.modules_repo, component, version, self.installed_by, install_track + self.component_type, ModulesRepo(remote_url), component, version, self.installed_by, install_track ) if self.component_type == "subworkflows": diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index dc47843601..bab977006f 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -1187,10 +1187,10 @@ def recreate_dependencies(self, repo, org, subworkflow): for dep_mod in dep_mods: if isinstance(dep_mod, dict): - component_name = list(dep_mod.keys())[0] - repo = dep_mod[component_name]["git_remote"] - org = dep_mod[component_name]["org_path"] - dep_mod = component_name + if dep_mod["git_remote"] is not None: + repo = dep_mod["git_remote"] + org = dep_mod["org_path"] + dep_mod = dep_mod["name"] installed_by = self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"] if installed_by == ["modules"]: self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"] = [] From 1737c88e1f73e82229329fdd0f0a972e6467d746 Mon Sep 17 00:00:00 2001 From: jvfe Date: Mon, 15 Jul 2024 09:59:18 -0300 Subject: [PATCH 08/55] fix: Add correct return type --- nf_core/components/components_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 0697fcd15f..440e81ebd9 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -1,7 +1,7 @@ import logging import re from pathlib import Path -from typing import List, Optional, Tuple +from typing import Dict, List, Optional, Tuple import questionary import rich.prompt @@ -133,7 +133,7 @@ def prompt_component_version_sha( return git_sha -def get_components_to_install(subworkflow_dir: str) -> Tuple[List[dict], List[str]]: +def get_components_to_install(subworkflow_dir: str) -> Tuple[List[Dict[str, str | None]], List[str]]: """ Parse the subworkflow main.nf file to retrieve all imported modules and subworkflows. """ From f7aaea2a9b9b21af239f03eb7fa16b147a76e1b4 Mon Sep 17 00:00:00 2001 From: jvfe Date: Mon, 15 Jul 2024 10:03:59 -0300 Subject: [PATCH 09/55] fix: Use any for values --- nf_core/components/components_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 440e81ebd9..6de6e2069b 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -1,7 +1,7 @@ import logging import re from pathlib import Path -from typing import Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple import questionary import rich.prompt @@ -133,7 +133,7 @@ def prompt_component_version_sha( return git_sha -def get_components_to_install(subworkflow_dir: str) -> Tuple[List[Dict[str, str | None]], List[str]]: +def get_components_to_install(subworkflow_dir: str) -> Tuple[List[Dict[str, Any]], List[str]]: """ Parse the subworkflow main.nf file to retrieve all imported modules and subworkflows. """ From d1c490d864f45ed423e927fbc1a700263d97e902 Mon Sep 17 00:00:00 2001 From: jvfe Date: Mon, 15 Jul 2024 10:22:44 -0300 Subject: [PATCH 10/55] test: Fix module key in across orgs --- tests/subworkflows/install.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index f872b1f7fb..614ff33845 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -8,7 +8,6 @@ from ..utils import ( CROSS_ORGANIZATION_URL, GITLAB_BRANCH_TEST_BRANCH, - GITLAB_DEFAULT_BRANCH, GITLAB_REPO, GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH, @@ -83,13 +82,13 @@ def test_subworkflows_install_different_branch_fail(self): def test_subworkflows_install_across_organizations(self): """Test installing a subworkflow with modules from different organizations""" - install_obj = SubworkflowInstall(self.pipeline_dir, remote_url=CROSS_ORGANIZATION_URL, branch=GITLAB_DEFAULT_BRANCH) + install_obj = SubworkflowInstall(self.pipeline_dir, remote_url=CROSS_ORGANIZATION_URL) # The hic_bwamem2 subworkflow contains modules from different organizations install_obj.install("get_genome_annotation") # Verify that the installed_by entry was added correctly modules_json = ModulesJson(self.pipeline_dir) mod_json = modules_json.get_modules_json() - assert mod_json["repos"][CROSS_ORGANIZATION_URL]["modules"]["jvfe"]["prokka"]["installed_by"] == [ + assert mod_json["repos"][CROSS_ORGANIZATION_URL]["modules"]["jvfe"]["wget"]["installed_by"] == [ "get_genome_annotation" ] From 6fa5cf49722f203755e30ede69e58594d0370316 Mon Sep 17 00:00:00 2001 From: jvfe Date: Mon, 15 Jul 2024 10:53:50 -0300 Subject: [PATCH 11/55] refact: Reset modules repo when git_remote not defined --- nf_core/components/install.py | 36 ++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index d76a65bf45..1d70ab5139 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -53,12 +53,12 @@ def install(self, component, silent=False): # Check modules directory structure self.check_modules_structure() - repo_path = self.modules_repo.repo_path - remote_url = self.modules_repo.remote_url if isinstance(component, dict): if component["git_remote"] is not None: - repo_path = component["org_path"] remote_url = component["git_remote"] + self.modules_repo = ModulesRepo(remote_url) + else: + self.modules_repo = ModulesRepo() component = component["name"] # Verify that 'modules.json' is consistent with the installed modules and subworkflows @@ -73,59 +73,61 @@ def install(self, component, silent=False): return False # Verify SHA - if not ModulesRepo(remote_url).verify_sha(self.prompt, self.sha): + if not self.modules_repo.verify_sha(self.prompt, self.sha): return False # Check and verify component name - component = self.collect_and_verify_name(component, ModulesRepo(remote_url)) + component = self.collect_and_verify_name(component, self.modules_repo) if not component: return False # Get current version - current_version = modules_json.get_component_version(self.component_type, component, remote_url, repo_path) + current_version = modules_json.get_component_version( + self.component_type, component, self.modules_repo.remote_url, self.modules_repo.repo_path + ) # Set the install folder based on the repository name - install_folder = Path(self.dir, self.component_type, repo_path) + install_folder = Path(self.dir, self.component_type, self.modules_repo.repo_path) # Compute the component directory component_dir = Path(install_folder, component) # Check that the component is not already installed component_not_installed = self.check_component_installed( - component, current_version, component_dir, ModulesRepo(remote_url), self.force, self.prompt, silent + component, current_version, component_dir, self.modules_repo, self.force, self.prompt, silent ) if not component_not_installed: log.debug( f"{self.component_type[:-1].title()} is already installed and force is not set.\nAdding the new installation source {self.installed_by} for {self.component_type[:-1]} {component} to 'modules.json' without installing the {self.component_type}." ) modules_json.load() - modules_json.update( - self.component_type, ModulesRepo(remote_url), component, current_version, self.installed_by - ) + modules_json.update(self.component_type, self.modules_repo, component, current_version, self.installed_by) return False - version = self.get_version(component, self.sha, self.prompt, current_version, ModulesRepo(remote_url)) + version = self.get_version(component, self.sha, self.prompt, current_version, self.modules_repo) if not version: return False # Remove component if force is set and component is installed install_track = None if self.force: - log.debug(f"Removing installed version of '{repo_path}/{component}'") + log.debug(f"Removing installed version of '{self.modules_repo.repo_path}/{component}'") self.clear_component_dir(component, component_dir) - install_track = self.clean_modules_json(component, ModulesRepo(remote_url), modules_json) + install_track = self.clean_modules_json(component, self.modules_repo, modules_json) if not silent: log.info(f"{'Rei' if self.force else 'I'}nstalling '{component}'") - log.debug(f"Installing {self.component_type} '{component}' at modules hash {version} from {remote_url}") + log.debug( + f"Installing {self.component_type} '{component}' at modules hash {version} from {self.modules_repo.remote_url}" + ) # Download component files - if not self.install_component_files(component, version, ModulesRepo(remote_url), install_folder): + if not self.install_component_files(component, version, self.modules_repo, install_folder): return False # Update module.json with newly installed subworkflow modules_json.load() modules_json.update( - self.component_type, ModulesRepo(remote_url), component, version, self.installed_by, install_track + self.component_type, self.modules_repo, component, version, self.installed_by, install_track ) if self.component_type == "subworkflows": From d9032820a2b2fc9a09757a8686ba32bd33fe1e28 Mon Sep 17 00:00:00 2001 From: jvfe Date: Mon, 15 Jul 2024 11:40:29 -0300 Subject: [PATCH 12/55] refact: Copy parent attribute --- nf_core/components/install.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 1d70ab5139..3acde5f49f 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -33,6 +33,7 @@ def __init__( installed_by=False, ): super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) + self.current_remote = remote_url self.force = force self.prompt = prompt self.sha = sha @@ -58,7 +59,7 @@ def install(self, component, silent=False): remote_url = component["git_remote"] self.modules_repo = ModulesRepo(remote_url) else: - self.modules_repo = ModulesRepo() + self.modules_repo = ModulesRepo(self.current_remote) component = component["name"] # Verify that 'modules.json' is consistent with the installed modules and subworkflows From 7f095fac8c40ad203fe6bcc803f34d46c15515a9 Mon Sep 17 00:00:00 2001 From: jvfe Date: Mon, 15 Jul 2024 14:32:13 -0300 Subject: [PATCH 13/55] refact: Keep old strategy as fallback --- nf_core/components/components_utils.py | 41 +++++++++++++++----------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 6de6e2069b..5dec79ef0f 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -147,22 +147,29 @@ def get_components_to_install(subworkflow_dir: str) -> Tuple[List[Dict[str, Any] match = regex.match(line) if match and len(match.groups()) == 2: name, link = match.groups() - if link.startswith("../") and not link.startswith("../../"): + if link.startswith("../../../"): + name_split = name.lower().split("_") + modules.append({"name": "/".join(name_split), "org_path": None, "git_remote": None}) + elif link.startswith("../"): subworkflows.append(name.lower()) - with open(Path(subworkflow_dir, "meta.yml")) as fh: - meta = yaml.safe_load(fh) - components = meta.get("components") - component_list = [] - for component in components: - if isinstance(component, str): - comp_dict = {"name": component, "org_path": None, "git_remote": None} - else: - name = list(component.keys())[0] - comp_dict = { - "name": name, - "org_path": component[name]["org_path"], - "git_remote": component[name]["git_remote"], - } - component_list.append(comp_dict) - modules.extend(component_list) + + if Path(subworkflow_dir, "meta.yml").exists(): + with open(Path(subworkflow_dir, "meta.yml")) as fh: + meta = yaml.safe_load(fh) + if "components" not in meta: + return modules, subworkflows + components = meta.get("components") + component_list = [] + for component in components: + if isinstance(component, str): + comp_dict = {"name": component, "org_path": None, "git_remote": None} + else: + name = list(component.keys())[0] + comp_dict = { + "name": name, + "org_path": component[name]["org_path"], + "git_remote": component[name]["git_remote"], + } + component_list.append(comp_dict) + modules.extend(component_list) return modules, subworkflows From 012e9d6d0e618dd3eef7aa9195c6663fd7482d6e Mon Sep 17 00:00:00 2001 From: jvfe Date: Mon, 15 Jul 2024 15:46:08 -0300 Subject: [PATCH 14/55] refact: Check component not in subwf list --- nf_core/components/components_utils.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 5dec79ef0f..ef2fd5b16f 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -161,15 +161,16 @@ def get_components_to_install(subworkflow_dir: str) -> Tuple[List[Dict[str, Any] components = meta.get("components") component_list = [] for component in components: - if isinstance(component, str): - comp_dict = {"name": component, "org_path": None, "git_remote": None} - else: - name = list(component.keys())[0] - comp_dict = { - "name": name, - "org_path": component[name]["org_path"], - "git_remote": component[name]["git_remote"], - } - component_list.append(comp_dict) + if component not in subworkflows: + if isinstance(component, str): + comp_dict = {"name": component, "org_path": None, "git_remote": None} + else: + name = list(component.keys())[0] + comp_dict = { + "name": name, + "org_path": component[name]["org_path"], + "git_remote": component[name]["git_remote"], + } + component_list.append(comp_dict) modules.extend(component_list) return modules, subworkflows From d77fccb85ae7f6d49224b96a75648425a2518bd9 Mon Sep 17 00:00:00 2001 From: jvfe Date: Wed, 24 Jul 2024 10:13:23 -0300 Subject: [PATCH 15/55] refact: Change return type to optional --- nf_core/components/components_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index ef2fd5b16f..1fbce9f03d 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -1,7 +1,7 @@ import logging import re from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple import questionary import rich.prompt @@ -133,7 +133,7 @@ def prompt_component_version_sha( return git_sha -def get_components_to_install(subworkflow_dir: str) -> Tuple[List[Dict[str, Any]], List[str]]: +def get_components_to_install(subworkflow_dir: str) -> Tuple[List[Dict[str, Optional[str]]], List[str]]: """ Parse the subworkflow main.nf file to retrieve all imported modules and subworkflows. """ @@ -154,6 +154,7 @@ def get_components_to_install(subworkflow_dir: str) -> Tuple[List[Dict[str, Any] subworkflows.append(name.lower()) if Path(subworkflow_dir, "meta.yml").exists(): + modules = [] with open(Path(subworkflow_dir, "meta.yml")) as fh: meta = yaml.safe_load(fh) if "components" not in meta: From faa7faf3051d0b5d9ce4fc8f543dba30d5a8c3dd Mon Sep 17 00:00:00 2001 From: jvfe Date: Wed, 24 Jul 2024 10:33:58 -0300 Subject: [PATCH 16/55] refact: Change way of handling dicts in modulesjson --- nf_core/modules/modules_json.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index bab977006f..c68cecf9db 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -1187,11 +1187,17 @@ def recreate_dependencies(self, repo, org, subworkflow): for dep_mod in dep_mods: if isinstance(dep_mod, dict): + name = dep_mod["name"] if dep_mod["git_remote"] is not None: - repo = dep_mod["git_remote"] - org = dep_mod["org_path"] - dep_mod = dep_mod["name"] - installed_by = self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"] + current_repo = dep_mod["git_remote"] + current_org = dep_mod["org_path"] + installed_by = self.modules_json["repos"][current_repo]["modules"][current_org][name][ + "installed_by" + ] + else: + installed_by = self.modules_json["repos"][repo]["modules"][org][name]["installed_by"] + else: + installed_by = self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"] if installed_by == ["modules"]: self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"] = [] if subworkflow not in installed_by: From 0f08bbe73b5ad4ce0629490bed660a29cee1f669 Mon Sep 17 00:00:00 2001 From: jvfe Date: Wed, 24 Jul 2024 11:13:39 -0300 Subject: [PATCH 17/55] refact: Handle dicts in meta_yml lint --- nf_core/subworkflows/lint/meta_yml.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nf_core/subworkflows/lint/meta_yml.py b/nf_core/subworkflows/lint/meta_yml.py index 24e75eddbf..9dbe517ec2 100644 --- a/nf_core/subworkflows/lint/meta_yml.py +++ b/nf_core/subworkflows/lint/meta_yml.py @@ -93,6 +93,9 @@ def meta_yml(subworkflow_lint_object, subworkflow): included_components = ( included_components[0] + included_components[1] ) # join included modules and included subworkflows in a single list + included_components = [ + component["name"] if isinstance(component, dict) else component for component in included_components + ] if "components" in meta_yaml: meta_components = [x for x in meta_yaml["components"]] for component in set(included_components): From 6d052de0cae90cd0b3c086e3e6042dc4894a15f5 Mon Sep 17 00:00:00 2001 From: jvfe Date: Sun, 28 Jul 2024 11:23:43 -0300 Subject: [PATCH 18/55] fix: Pass branch in install too --- nf_core/components/install.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 3acde5f49f..6fbc9d0a9d 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -34,6 +34,7 @@ def __init__( ): super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) self.current_remote = remote_url + self.branch = branch self.force = force self.prompt = prompt self.sha = sha @@ -57,9 +58,9 @@ def install(self, component, silent=False): if isinstance(component, dict): if component["git_remote"] is not None: remote_url = component["git_remote"] - self.modules_repo = ModulesRepo(remote_url) + self.modules_repo = ModulesRepo(remote_url, self.branch) else: - self.modules_repo = ModulesRepo(self.current_remote) + self.modules_repo = ModulesRepo(self.current_remote, self.branch) component = component["name"] # Verify that 'modules.json' is consistent with the installed modules and subworkflows From c6ce67f00bc5a45493ec435f53d82e1bdc475adf Mon Sep 17 00:00:00 2001 From: jvfe Date: Mon, 5 Aug 2024 10:13:22 -0300 Subject: [PATCH 19/55] fix: Check if module in meta.yml is imported --- nf_core/components/components_utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 1fbce9f03d..3c62e0ab24 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -154,7 +154,6 @@ def get_components_to_install(subworkflow_dir: str) -> Tuple[List[Dict[str, Opti subworkflows.append(name.lower()) if Path(subworkflow_dir, "meta.yml").exists(): - modules = [] with open(Path(subworkflow_dir, "meta.yml")) as fh: meta = yaml.safe_load(fh) if "components" not in meta: @@ -162,7 +161,7 @@ def get_components_to_install(subworkflow_dir: str) -> Tuple[List[Dict[str, Opti components = meta.get("components") component_list = [] for component in components: - if component not in subworkflows: + if component not in subworkflows and component in [d["name"] for d in modules]: if isinstance(component, str): comp_dict = {"name": component, "org_path": None, "git_remote": None} else: @@ -173,5 +172,5 @@ def get_components_to_install(subworkflow_dir: str) -> Tuple[List[Dict[str, Opti "git_remote": component[name]["git_remote"], } component_list.append(comp_dict) - modules.extend(component_list) + modules = component_list return modules, subworkflows From fca6f279ba61fa6a441017cef21621c8bc8ff771 Mon Sep 17 00:00:00 2001 From: jvfe Date: Sat, 10 Aug 2024 14:45:22 -0300 Subject: [PATCH 20/55] refact: Define org path based on git remote --- nf_core/components/components_utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 3c62e0ab24..892ddb53db 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -166,10 +166,14 @@ def get_components_to_install(subworkflow_dir: str) -> Tuple[List[Dict[str, Opti comp_dict = {"name": component, "org_path": None, "git_remote": None} else: name = list(component.keys())[0] + git_remote = component[name]["git_remote"] + match_name = re.match(r"(?:https://|git@)[\w\.]+[:/](.*?)/", git_remote) + if match_name is not None: + org_path = match_name.group(1) comp_dict = { "name": name, - "org_path": component[name]["org_path"], - "git_remote": component[name]["git_remote"], + "org_path": org_path, + "git_remote": git_remote, } component_list.append(comp_dict) modules = component_list From 092bf912d33642b061b23b5328909d7a98657aef Mon Sep 17 00:00:00 2001 From: jvfe Date: Sat, 10 Aug 2024 15:08:38 -0300 Subject: [PATCH 21/55] feat: Allow defining branches in meta.yml --- nf_core/components/components_utils.py | 1 + nf_core/components/install.py | 3 ++- tests/subworkflows/install.py | 5 ++++- tests/utils.py | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 892ddb53db..d24291fd95 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -174,6 +174,7 @@ def get_components_to_install(subworkflow_dir: str) -> Tuple[List[Dict[str, Opti "name": name, "org_path": org_path, "git_remote": git_remote, + "branch": component[name].get("branch", "master"), } component_list.append(comp_dict) modules = component_list diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 6fbc9d0a9d..2417eddfe3 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -58,7 +58,8 @@ def install(self, component, silent=False): if isinstance(component, dict): if component["git_remote"] is not None: remote_url = component["git_remote"] - self.modules_repo = ModulesRepo(remote_url, self.branch) + branch = component["branch"] + self.modules_repo = ModulesRepo(remote_url, branch) else: self.modules_repo = ModulesRepo(self.current_remote, self.branch) component = component["name"] diff --git a/tests/subworkflows/install.py b/tests/subworkflows/install.py index 614ff33845..45179e9e57 100644 --- a/tests/subworkflows/install.py +++ b/tests/subworkflows/install.py @@ -6,6 +6,7 @@ from nf_core.subworkflows.install import SubworkflowInstall from ..utils import ( + CROSS_ORGANIZATION_BRANCH, CROSS_ORGANIZATION_URL, GITLAB_BRANCH_TEST_BRANCH, GITLAB_REPO, @@ -82,7 +83,9 @@ def test_subworkflows_install_different_branch_fail(self): def test_subworkflows_install_across_organizations(self): """Test installing a subworkflow with modules from different organizations""" - install_obj = SubworkflowInstall(self.pipeline_dir, remote_url=CROSS_ORGANIZATION_URL) + install_obj = SubworkflowInstall( + self.pipeline_dir, remote_url=CROSS_ORGANIZATION_URL, branch=CROSS_ORGANIZATION_BRANCH + ) # The hic_bwamem2 subworkflow contains modules from different organizations install_obj.install("get_genome_annotation") # Verify that the installed_by entry was added correctly diff --git a/tests/utils.py b/tests/utils.py index 30dd1ab224..6cd7ed535f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -17,6 +17,7 @@ OLD_TRIMGALORE_BRANCH = "mimic-old-trimgalore" GITLAB_URL = "https://gitlab.com/nf-core/modules-test.git" CROSS_ORGANIZATION_URL = "https://github.com/jvfe/test-subworkflow-remote.git" +CROSS_ORGANIZATION_BRANCH = "main" GITLAB_REPO = "nf-core-test" GITLAB_DEFAULT_BRANCH = "main" GITLAB_SUBWORKFLOWS_BRANCH = "subworkflows" From 208d796c686cd2f944d7cfc7d368b17ed0cf37b4 Mon Sep 17 00:00:00 2001 From: jvfe Date: Sat, 10 Aug 2024 15:15:43 -0300 Subject: [PATCH 22/55] fix: Add empty branch in other dicts --- nf_core/components/components_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index d24291fd95..4a0f41b6b6 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -149,7 +149,7 @@ def get_components_to_install(subworkflow_dir: str) -> Tuple[List[Dict[str, Opti name, link = match.groups() if link.startswith("../../../"): name_split = name.lower().split("_") - modules.append({"name": "/".join(name_split), "org_path": None, "git_remote": None}) + modules.append({"name": "/".join(name_split), "org_path": None, "git_remote": None, "branch": None}) elif link.startswith("../"): subworkflows.append(name.lower()) @@ -163,7 +163,7 @@ def get_components_to_install(subworkflow_dir: str) -> Tuple[List[Dict[str, Opti for component in components: if component not in subworkflows and component in [d["name"] for d in modules]: if isinstance(component, str): - comp_dict = {"name": component, "org_path": None, "git_remote": None} + comp_dict = {"name": component, "org_path": None, "git_remote": None, "branch": None} else: name = list(component.keys())[0] git_remote = component[name]["git_remote"] From 71c43bee6625428ebe98bf60f24239233fc0cea7 Mon Sep 17 00:00:00 2001 From: jvfe Date: Sun, 11 Aug 2024 21:52:11 -0300 Subject: [PATCH 23/55] refact: Rework logic to use subwfs as well --- nf_core/components/components_utils.py | 62 ++++++++++++++++---------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 4a0f41b6b6..411c18afe8 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -133,7 +133,9 @@ def prompt_component_version_sha( return git_sha -def get_components_to_install(subworkflow_dir: str) -> Tuple[List[Dict[str, Optional[str]]], List[str]]: +def get_components_to_install( + subworkflow_dir: str, +) -> Tuple[List[Dict[str, Optional[str]]], List[Dict[str, Optional[str]]]]: """ Parse the subworkflow main.nf file to retrieve all imported modules and subworkflows. """ @@ -149,33 +151,45 @@ def get_components_to_install(subworkflow_dir: str) -> Tuple[List[Dict[str, Opti name, link = match.groups() if link.startswith("../../../"): name_split = name.lower().split("_") - modules.append({"name": "/".join(name_split), "org_path": None, "git_remote": None, "branch": None}) + component_dict = { + "name": "/".join(name_split), + "org_path": None, + "git_remote": None, + "branch": None, + } + modules.append(component_dict) elif link.startswith("../"): - subworkflows.append(name.lower()) + component_dict = {"name": name.lower(), "org_path": None, "git_remote": None, "branch": None} + subworkflows.append(component_dict) if Path(subworkflow_dir, "meta.yml").exists(): with open(Path(subworkflow_dir, "meta.yml")) as fh: meta = yaml.safe_load(fh) - if "components" not in meta: - return modules, subworkflows - components = meta.get("components") - component_list = [] + components = meta.get("components", []) + new_module_list = [] + new_subwf_list = [] + for component in components: - if component not in subworkflows and component in [d["name"] for d in modules]: - if isinstance(component, str): - comp_dict = {"name": component, "org_path": None, "git_remote": None, "branch": None} - else: - name = list(component.keys())[0] - git_remote = component[name]["git_remote"] - match_name = re.match(r"(?:https://|git@)[\w\.]+[:/](.*?)/", git_remote) - if match_name is not None: - org_path = match_name.group(1) - comp_dict = { - "name": name, - "org_path": org_path, - "git_remote": git_remote, - "branch": component[name].get("branch", "master"), - } - component_list.append(comp_dict) - modules = component_list + if isinstance(component, str): + component_dict = {"name": component, "org_path": None, "git_remote": None, "branch": None} + else: + name = list(component.keys())[0] + git_remote = component[name]["git_remote"] + org_path_match = re.search(r"(?:https://|git@)[\w\.]+[:/](.*?)/", git_remote) + org_path = org_path_match.group(1) if org_path_match else None + component_dict = { + "name": name.lower(), + "org_path": org_path, + "git_remote": git_remote, + "branch": component[name].get("branch", "master"), + } + + if component_dict["name"] in [sw["name"] for sw in subworkflows]: + new_subwf_list.append(component_dict) + else: + new_module_list.append(component_dict) + + modules = new_module_list + subworkflows = new_subwf_list + return modules, subworkflows From 8bcf7374a8015a7aee0cc77fcd45c056b452b203 Mon Sep 17 00:00:00 2001 From: jvfe Date: Sun, 11 Aug 2024 22:14:00 -0300 Subject: [PATCH 24/55] refact: Support subwf dict in recreate deps --- nf_core/modules/modules_json.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 4436fab605..ba36d80632 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -1205,7 +1205,16 @@ def recreate_dependencies(self, repo, org, subworkflow): self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"].append(subworkflow) for dep_subwf in dep_subwfs: - installed_by = self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["installed_by"] + if isinstance(dep_subwf, dict): + name = dep_subwf["name"] + if dep_subwf["git_remote"] is not None: + current_repo = dep_subwf["git_remote"] + current_org = dep_subwf["org_path"] + installed_by = self.modules_json["repos"][current_repo]["modules"][current_org][name][ + "installed_by" + ] + else: + installed_by = self.modules_json["repos"][repo]["modules"][org][name]["installed_by"] if installed_by == ["subworkflows"]: self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["installed_by"] = [] if subworkflow not in installed_by: From efde7b76822a2393bddae0a92c389f96382223bf Mon Sep 17 00:00:00 2001 From: jvfe Date: Sun, 11 Aug 2024 22:26:10 -0300 Subject: [PATCH 25/55] fix: Change modules to subwfs --- nf_core/modules/modules_json.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index ba36d80632..7c40d0f86c 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -1210,11 +1210,11 @@ def recreate_dependencies(self, repo, org, subworkflow): if dep_subwf["git_remote"] is not None: current_repo = dep_subwf["git_remote"] current_org = dep_subwf["org_path"] - installed_by = self.modules_json["repos"][current_repo]["modules"][current_org][name][ + installed_by = self.modules_json["repos"][current_repo]["subworkflows"][current_org][name][ "installed_by" ] else: - installed_by = self.modules_json["repos"][repo]["modules"][org][name]["installed_by"] + installed_by = self.modules_json["repos"][repo]["subworkflows"][org][name]["installed_by"] if installed_by == ["subworkflows"]: self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["installed_by"] = [] if subworkflow not in installed_by: From 8e255875a6802f3800d52809221d29acbcb87a0a Mon Sep 17 00:00:00 2001 From: jvfe Date: Sun, 11 Aug 2024 22:43:48 -0300 Subject: [PATCH 26/55] fix: Use name value in recreate deps --- nf_core/modules/modules_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 7c40d0f86c..57ea70d3bb 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -1183,7 +1183,7 @@ def recreate_dependencies(self, repo, org, subworkflow): i.e., no module or subworkflow has been installed by the user in the meantime """ - sw_path = Path(self.subworkflows_dir, org, subworkflow) + sw_path = Path(self.subworkflows_dir, org, subworkflow["name"]) dep_mods, dep_subwfs = get_components_to_install(sw_path) for dep_mod in dep_mods: From 494791368b4cc6acafb81062d73ff61f9116e150 Mon Sep 17 00:00:00 2001 From: jvfe Date: Sun, 11 Aug 2024 22:48:48 -0300 Subject: [PATCH 27/55] Revert "fix: Use name value in recreate deps" This reverts commit 8e255875a6802f3800d52809221d29acbcb87a0a. --- nf_core/modules/modules_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 57ea70d3bb..7c40d0f86c 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -1183,7 +1183,7 @@ def recreate_dependencies(self, repo, org, subworkflow): i.e., no module or subworkflow has been installed by the user in the meantime """ - sw_path = Path(self.subworkflows_dir, org, subworkflow["name"]) + sw_path = Path(self.subworkflows_dir, org, subworkflow) dep_mods, dep_subwfs = get_components_to_install(sw_path) for dep_mod in dep_mods: From 4888d3251982337ba31a4b73de960cac7a3d8f3b Mon Sep 17 00:00:00 2001 From: jvfe Date: Mon, 12 Aug 2024 08:12:22 -0300 Subject: [PATCH 28/55] fix: Use subworkflow name in recreate deps --- nf_core/modules/modules_json.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 7c40d0f86c..fce8c630c0 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -1183,7 +1183,8 @@ def recreate_dependencies(self, repo, org, subworkflow): i.e., no module or subworkflow has been installed by the user in the meantime """ - sw_path = Path(self.subworkflows_dir, org, subworkflow) + sw_name = subworkflow["name"] if isinstance(subworkflow, dict) else subworkflow + sw_path = Path(self.subworkflows_dir, org, sw_name) dep_mods, dep_subwfs = get_components_to_install(sw_path) for dep_mod in dep_mods: From aa265d8fa0626b135b72be1db6b1cde7b3bbbeea Mon Sep 17 00:00:00 2001 From: jvfe Date: Mon, 12 Aug 2024 08:27:12 -0300 Subject: [PATCH 29/55] fix: Use sw_name in appends too --- nf_core/modules/modules_json.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index fce8c630c0..9e8e975fe5 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -1202,8 +1202,8 @@ def recreate_dependencies(self, repo, org, subworkflow): installed_by = self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"] if installed_by == ["modules"]: self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"] = [] - if subworkflow not in installed_by: - self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"].append(subworkflow) + if sw_name not in installed_by: + self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"].append(sw_name) for dep_subwf in dep_subwfs: if isinstance(dep_subwf, dict): @@ -1218,6 +1218,6 @@ def recreate_dependencies(self, repo, org, subworkflow): installed_by = self.modules_json["repos"][repo]["subworkflows"][org][name]["installed_by"] if installed_by == ["subworkflows"]: self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["installed_by"] = [] - if subworkflow not in installed_by: - self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["installed_by"].append(subworkflow) + if sw_name not in installed_by: + self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["installed_by"].append(sw_name) self.recreate_dependencies(repo, org, dep_subwf) From a764c76092fa7311378491d4f868a0e7fc290e6e Mon Sep 17 00:00:00 2001 From: jvfe Date: Mon, 12 Aug 2024 20:53:35 -0300 Subject: [PATCH 30/55] fix: Only add module if it's in main.nf too --- nf_core/components/components_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 411c18afe8..4faa9f149f 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -187,7 +187,8 @@ def get_components_to_install( if component_dict["name"] in [sw["name"] for sw in subworkflows]: new_subwf_list.append(component_dict) else: - new_module_list.append(component_dict) + if component_dict["name"] in [m["name"] for m in modules]: + new_module_list.append(component_dict) modules = new_module_list subworkflows = new_subwf_list From 869d8165be959be82fcefe808ccfe75cfe09f8e2 Mon Sep 17 00:00:00 2001 From: jvfe Date: Tue, 13 Aug 2024 09:05:54 -0300 Subject: [PATCH 31/55] fix: Handle incomplete meta.yml --- nf_core/components/components_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 4faa9f149f..eab9390321 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -165,7 +165,9 @@ def get_components_to_install( if Path(subworkflow_dir, "meta.yml").exists(): with open(Path(subworkflow_dir, "meta.yml")) as fh: meta = yaml.safe_load(fh) - components = meta.get("components", []) + if "components" not in meta: + return modules, subworkflows + components = meta.get("components") new_module_list = [] new_subwf_list = [] From c3ac1b8ab5da6c4c98fee9804575b15ea6b18554 Mon Sep 17 00:00:00 2001 From: jvfe Date: Tue, 13 Aug 2024 11:23:43 -0300 Subject: [PATCH 32/55] refact: Remove isinstance check in lint/meta.yml --- nf_core/subworkflows/lint/meta_yml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/subworkflows/lint/meta_yml.py b/nf_core/subworkflows/lint/meta_yml.py index 9dbe517ec2..9b42b19957 100644 --- a/nf_core/subworkflows/lint/meta_yml.py +++ b/nf_core/subworkflows/lint/meta_yml.py @@ -94,7 +94,7 @@ def meta_yml(subworkflow_lint_object, subworkflow): included_components[0] + included_components[1] ) # join included modules and included subworkflows in a single list included_components = [ - component["name"] if isinstance(component, dict) else component for component in included_components + component["name"] for component in included_components ] if "components" in meta_yaml: meta_components = [x for x in meta_yaml["components"]] From 4eb95e6c6ce3eeba0bf519e4a49ef3174c0b3d7f Mon Sep 17 00:00:00 2001 From: jvfe Date: Tue, 13 Aug 2024 11:28:21 -0300 Subject: [PATCH 33/55] style: Format meta_yml.py --- nf_core/subworkflows/lint/meta_yml.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nf_core/subworkflows/lint/meta_yml.py b/nf_core/subworkflows/lint/meta_yml.py index 9b42b19957..9c96df7563 100644 --- a/nf_core/subworkflows/lint/meta_yml.py +++ b/nf_core/subworkflows/lint/meta_yml.py @@ -93,9 +93,7 @@ def meta_yml(subworkflow_lint_object, subworkflow): included_components = ( included_components[0] + included_components[1] ) # join included modules and included subworkflows in a single list - included_components = [ - component["name"] for component in included_components - ] + included_components = [component["name"] for component in included_components] if "components" in meta_yaml: meta_components = [x for x in meta_yaml["components"]] for component in set(included_components): From e9bf23830492483e4ef9b438603dd60cf5ac8af0 Mon Sep 17 00:00:00 2001 From: jvfe Date: Tue, 13 Aug 2024 11:33:04 -0300 Subject: [PATCH 34/55] refact: Remove isinstance check in components/install.py --- nf_core/components/install.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 2417eddfe3..b151ab427d 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -44,8 +44,10 @@ def __init__( self.installed_by = self.component_type def install(self, component, silent=False): + component_name = component["name"] + if self.repo_type == "modules": - log.error(f"You cannot install a {component} in a clone of nf-core/modules") + log.error(f"You cannot install a {component_name} in a clone of nf-core/modules") return False # Check whether pipelines is valid if not self.has_valid_directory(): @@ -55,14 +57,12 @@ def install(self, component, silent=False): # Check modules directory structure self.check_modules_structure() - if isinstance(component, dict): - if component["git_remote"] is not None: - remote_url = component["git_remote"] - branch = component["branch"] - self.modules_repo = ModulesRepo(remote_url, branch) - else: - self.modules_repo = ModulesRepo(self.current_remote, self.branch) - component = component["name"] + if component["git_remote"] is not None: + remote_url = component["git_remote"] + branch = component["branch"] + self.modules_repo = ModulesRepo(remote_url, branch) + else: + self.modules_repo = ModulesRepo(self.current_remote, self.branch) # Verify that 'modules.json' is consistent with the installed modules and subworkflows modules_json = ModulesJson(self.dir) @@ -80,7 +80,7 @@ def install(self, component, silent=False): return False # Check and verify component name - component = self.collect_and_verify_name(component, self.modules_repo) + component = self.collect_and_verify_name(component_name, self.modules_repo) if not component: return False From c8b8a83f40106ad76a1699a5f3f50880f8a9aec3 Mon Sep 17 00:00:00 2001 From: jvfe Date: Tue, 13 Aug 2024 12:46:16 -0300 Subject: [PATCH 35/55] Revert "refact: Remove isinstance check in components/install.py" This reverts commit e9bf23830492483e4ef9b438603dd60cf5ac8af0. --- nf_core/components/install.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index b151ab427d..2417eddfe3 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -44,10 +44,8 @@ def __init__( self.installed_by = self.component_type def install(self, component, silent=False): - component_name = component["name"] - if self.repo_type == "modules": - log.error(f"You cannot install a {component_name} in a clone of nf-core/modules") + log.error(f"You cannot install a {component} in a clone of nf-core/modules") return False # Check whether pipelines is valid if not self.has_valid_directory(): @@ -57,12 +55,14 @@ def install(self, component, silent=False): # Check modules directory structure self.check_modules_structure() - if component["git_remote"] is not None: - remote_url = component["git_remote"] - branch = component["branch"] - self.modules_repo = ModulesRepo(remote_url, branch) - else: - self.modules_repo = ModulesRepo(self.current_remote, self.branch) + if isinstance(component, dict): + if component["git_remote"] is not None: + remote_url = component["git_remote"] + branch = component["branch"] + self.modules_repo = ModulesRepo(remote_url, branch) + else: + self.modules_repo = ModulesRepo(self.current_remote, self.branch) + component = component["name"] # Verify that 'modules.json' is consistent with the installed modules and subworkflows modules_json = ModulesJson(self.dir) @@ -80,7 +80,7 @@ def install(self, component, silent=False): return False # Check and verify component name - component = self.collect_and_verify_name(component_name, self.modules_repo) + component = self.collect_and_verify_name(component, self.modules_repo) if not component: return False From eba839167edfb4de42dd172cb872893b700855d0 Mon Sep 17 00:00:00 2001 From: jvfe Date: Tue, 13 Aug 2024 16:06:09 -0300 Subject: [PATCH 36/55] refact: Remove isinstance check in recreate_deps --- nf_core/modules/modules_json.py | 36 ++++++++++++++------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 9e8e975fe5..b2c2afe897 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -1188,34 +1188,28 @@ def recreate_dependencies(self, repo, org, subworkflow): dep_mods, dep_subwfs = get_components_to_install(sw_path) for dep_mod in dep_mods: - if isinstance(dep_mod, dict): - name = dep_mod["name"] - if dep_mod["git_remote"] is not None: - current_repo = dep_mod["git_remote"] - current_org = dep_mod["org_path"] - installed_by = self.modules_json["repos"][current_repo]["modules"][current_org][name][ - "installed_by" - ] - else: - installed_by = self.modules_json["repos"][repo]["modules"][org][name]["installed_by"] + name = dep_mod["name"] + if dep_mod["git_remote"] is not None: + current_repo = dep_mod["git_remote"] + current_org = dep_mod["org_path"] + installed_by = self.modules_json["repos"][current_repo]["modules"][current_org][name]["installed_by"] else: - installed_by = self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"] + installed_by = self.modules_json["repos"][repo]["modules"][org][name]["installed_by"] if installed_by == ["modules"]: self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"] = [] if sw_name not in installed_by: self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"].append(sw_name) for dep_subwf in dep_subwfs: - if isinstance(dep_subwf, dict): - name = dep_subwf["name"] - if dep_subwf["git_remote"] is not None: - current_repo = dep_subwf["git_remote"] - current_org = dep_subwf["org_path"] - installed_by = self.modules_json["repos"][current_repo]["subworkflows"][current_org][name][ - "installed_by" - ] - else: - installed_by = self.modules_json["repos"][repo]["subworkflows"][org][name]["installed_by"] + name = dep_subwf["name"] + if dep_subwf["git_remote"] is not None: + current_repo = dep_subwf["git_remote"] + current_org = dep_subwf["org_path"] + installed_by = self.modules_json["repos"][current_repo]["subworkflows"][current_org][name][ + "installed_by" + ] + else: + installed_by = self.modules_json["repos"][repo]["subworkflows"][org][name]["installed_by"] if installed_by == ["subworkflows"]: self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["installed_by"] = [] if sw_name not in installed_by: From c72b94a3e6db1e06b1330c5e2da016382bada85b Mon Sep 17 00:00:00 2001 From: jvfe Date: Thu, 15 Aug 2024 14:06:16 -0300 Subject: [PATCH 37/55] refact: Change function structure to use dicts not lists --- nf_core/components/components_utils.py | 65 +++++++++++--------------- 1 file changed, 28 insertions(+), 37 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index d327d28af3..edd560b616 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -149,8 +149,8 @@ def get_components_to_install( """ Parse the subworkflow main.nf file to retrieve all imported modules and subworkflows. """ - modules = [] - subworkflows = [] + modules = {} + subworkflows = {} with open(Path(subworkflow_dir, "main.nf")) as fh: for line in fh: regex = re.compile( @@ -161,48 +161,39 @@ def get_components_to_install( name, link = match.groups() if link.startswith("../../../"): name_split = name.lower().split("_") + component_name = "/".join(name_split) component_dict = { - "name": "/".join(name_split), + "name": component_name, "org_path": None, "git_remote": None, "branch": None, } - modules.append(component_dict) + modules[component_dict[component_name]] = component_dict elif link.startswith("../"): - component_dict = {"name": name.lower(), "org_path": None, "git_remote": None, "branch": None} - subworkflows.append(component_dict) + component_name = name.lower() + component_dict = {"name": component_name, "org_path": None, "git_remote": None, "branch": None} + subworkflows[component_dict[component_name]] = component_dict if Path(subworkflow_dir, "meta.yml").exists(): with open(Path(subworkflow_dir, "meta.yml")) as fh: meta = yaml.safe_load(fh) - if "components" not in meta: - return modules, subworkflows - components = meta.get("components") - new_module_list = [] - new_subwf_list = [] - - for component in components: - if isinstance(component, str): - component_dict = {"name": component, "org_path": None, "git_remote": None, "branch": None} - else: - name = list(component.keys())[0] - git_remote = component[name]["git_remote"] - org_path_match = re.search(r"(?:https://|git@)[\w\.]+[:/](.*?)/", git_remote) - org_path = org_path_match.group(1) if org_path_match else None - component_dict = { - "name": name.lower(), - "org_path": org_path, - "git_remote": git_remote, - "branch": component[name].get("branch", "master"), - } - - if component_dict["name"] in [sw["name"] for sw in subworkflows]: - new_subwf_list.append(component_dict) - else: - if component_dict["name"] in [m["name"] for m in modules]: - new_module_list.append(component_dict) - - modules = new_module_list - subworkflows = new_subwf_list - - return modules, subworkflows + if "components" in meta: + components = meta.get("components") + for component in components: + if isinstance(component, dict): + component_name = list(component.keys())[0].lower() + git_remote = component[component_name]["git_remote"] + org_path_match = re.search(r"(?:https://|git@)[\w\.]+[:/](.*?)/", git_remote) + org_path = org_path_match.group(1) if org_path_match else None + current_comp_dict = subworkflows if component_name in subworkflows else modules + + component_dict = { + "name": component_name, + "org_path": org_path, + "git_remote": git_remote, + "branch": component[component_name].get("branch", "master"), + } + + current_comp_dict[component_dict[component_name]].update(component_dict) + + return list(modules.values()), list(subworkflows.values()) From bb31d78fd4a120d1df06d5933bd69b7d771de803 Mon Sep 17 00:00:00 2001 From: jvfe Date: Thu, 15 Aug 2024 14:46:33 -0300 Subject: [PATCH 38/55] fix: Change access key in inner dicts --- nf_core/components/components_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index edd560b616..0f02305077 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -168,11 +168,11 @@ def get_components_to_install( "git_remote": None, "branch": None, } - modules[component_dict[component_name]] = component_dict + modules[component_dict["name"]] = component_dict elif link.startswith("../"): component_name = name.lower() component_dict = {"name": component_name, "org_path": None, "git_remote": None, "branch": None} - subworkflows[component_dict[component_name]] = component_dict + subworkflows[component_dict["name"]] = component_dict if Path(subworkflow_dir, "meta.yml").exists(): with open(Path(subworkflow_dir, "meta.yml")) as fh: @@ -194,6 +194,6 @@ def get_components_to_install( "branch": component[component_name].get("branch", "master"), } - current_comp_dict[component_dict[component_name]].update(component_dict) + current_comp_dict[component_dict["name"]].update(component_dict) return list(modules.values()), list(subworkflows.values()) From 7f4cce83bb06c7de29edf603c0c9323e0a87a251 Mon Sep 17 00:00:00 2001 From: jvfe Date: Fri, 16 Aug 2024 11:19:52 -0300 Subject: [PATCH 39/55] refact: Use component_name every time --- nf_core/components/components_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 0f02305077..13c1fe1e0c 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -168,11 +168,11 @@ def get_components_to_install( "git_remote": None, "branch": None, } - modules[component_dict["name"]] = component_dict + modules[component_name] = component_dict elif link.startswith("../"): component_name = name.lower() component_dict = {"name": component_name, "org_path": None, "git_remote": None, "branch": None} - subworkflows[component_dict["name"]] = component_dict + subworkflows[component_name] = component_dict if Path(subworkflow_dir, "meta.yml").exists(): with open(Path(subworkflow_dir, "meta.yml")) as fh: @@ -194,6 +194,6 @@ def get_components_to_install( "branch": component[component_name].get("branch", "master"), } - current_comp_dict[component_dict["name"]].update(component_dict) + current_comp_dict[component_name].update(component_dict) return list(modules.values()), list(subworkflows.values()) From e342678e99b020ab3a9b4f31c61b1c05a1858568 Mon Sep 17 00:00:00 2001 From: jvfe Date: Fri, 16 Aug 2024 11:20:09 -0300 Subject: [PATCH 40/55] refact: Don't default to master --- nf_core/components/components_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 13c1fe1e0c..c8abe2d785 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -191,7 +191,7 @@ def get_components_to_install( "name": component_name, "org_path": org_path, "git_remote": git_remote, - "branch": component[component_name].get("branch", "master"), + "branch": component[component_name].get("branch", None), } current_comp_dict[component_name].update(component_dict) From 50216c45590be387eda69b77ea8a09e499e1a91b Mon Sep 17 00:00:00 2001 From: jvfe Date: Tue, 20 Aug 2024 08:01:28 -0300 Subject: [PATCH 41/55] refact: Move instance check up --- nf_core/components/install.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index ca746098dc..b3ad7749ed 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -50,6 +50,15 @@ def __init__( self.installed_by = [self.component_type] def install(self, component: str, silent: bool = False) -> bool: + if isinstance(component, dict): + if component["git_remote"] is not None: + remote_url = component["git_remote"] + branch = component["branch"] + self.modules_repo = ModulesRepo(remote_url, branch) + else: + self.modules_repo = ModulesRepo(self.current_remote, self.branch) + component = component["name"] + if self.repo_type == "modules": log.error(f"You cannot install a {component} in a clone of nf-core/modules") return False @@ -61,15 +70,6 @@ def install(self, component: str, silent: bool = False) -> bool: # Check modules directory structure self.check_modules_structure() - if isinstance(component, dict): - if component["git_remote"] is not None: - remote_url = component["git_remote"] - branch = component["branch"] - self.modules_repo = ModulesRepo(remote_url, branch) - else: - self.modules_repo = ModulesRepo(self.current_remote, self.branch) - component = component["name"] - # Verify that 'modules.json' is consistent with the installed modules and subworkflows modules_json = ModulesJson(self.directory) if not silent: From 0e83d9afbbf461a20abc65b1350cb95e9dfd8bfd Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Tue, 20 Aug 2024 13:23:56 +0100 Subject: [PATCH 42/55] Refactoring of component_utils.py --- nf_core/components/components_utils.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index c8abe2d785..e2d47fe65a 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -162,17 +162,12 @@ def get_components_to_install( if link.startswith("../../../"): name_split = name.lower().split("_") component_name = "/".join(name_split) - component_dict = { - "name": component_name, - "org_path": None, - "git_remote": None, - "branch": None, - } - modules[component_name] = component_dict + current_comp_dict = modules elif link.startswith("../"): component_name = name.lower() - component_dict = {"name": component_name, "org_path": None, "git_remote": None, "branch": None} - subworkflows[component_name] = component_dict + current_comp_dict = subworkflows + component_dict = {"name": component_name, "org_path": None, "git_remote": None, "branch": None} + current_comp_dict[component_name] = component_dict if Path(subworkflow_dir, "meta.yml").exists(): with open(Path(subworkflow_dir, "meta.yml")) as fh: From f8c8251c44fd469a4d305459b7b4cc0c38ffe24e Mon Sep 17 00:00:00 2001 From: jvfe Date: Tue, 20 Aug 2024 20:53:15 -0300 Subject: [PATCH 43/55] Revert "Refactoring of component_utils.py" This reverts commit 0e83d9afbbf461a20abc65b1350cb95e9dfd8bfd. --- nf_core/components/components_utils.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index e2d47fe65a..c8abe2d785 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -162,12 +162,17 @@ def get_components_to_install( if link.startswith("../../../"): name_split = name.lower().split("_") component_name = "/".join(name_split) - current_comp_dict = modules + component_dict = { + "name": component_name, + "org_path": None, + "git_remote": None, + "branch": None, + } + modules[component_name] = component_dict elif link.startswith("../"): component_name = name.lower() - current_comp_dict = subworkflows - component_dict = {"name": component_name, "org_path": None, "git_remote": None, "branch": None} - current_comp_dict[component_name] = component_dict + component_dict = {"name": component_name, "org_path": None, "git_remote": None, "branch": None} + subworkflows[component_name] = component_dict if Path(subworkflow_dir, "meta.yml").exists(): with open(Path(subworkflow_dir, "meta.yml")) as fh: From 313853ec737a484b6d05b1ae31ad1f04bd7ff980 Mon Sep 17 00:00:00 2001 From: jvfe Date: Wed, 21 Aug 2024 09:47:46 -0300 Subject: [PATCH 44/55] refact: Use get in components/install.py --- nf_core/components/install.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index b3ad7749ed..be3d8a40f7 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -51,13 +51,13 @@ def __init__( def install(self, component: str, silent: bool = False) -> bool: if isinstance(component, dict): - if component["git_remote"] is not None: - remote_url = component["git_remote"] - branch = component["branch"] + if component.get("git_remote") is not None: + remote_url = component.get("git_remote") + branch = component.get("branch") self.modules_repo = ModulesRepo(remote_url, branch) else: self.modules_repo = ModulesRepo(self.current_remote, self.branch) - component = component["name"] + component = component.get("name") if self.repo_type == "modules": log.error(f"You cannot install a {component} in a clone of nf-core/modules") From e8e4da0ee0223b05324b8cefc9365e0e31be9ee9 Mon Sep 17 00:00:00 2001 From: jvfe Date: Wed, 21 Aug 2024 11:30:16 -0300 Subject: [PATCH 45/55] refact: Avoid redefining keys when possible --- nf_core/components/components_utils.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index c8abe2d785..78c1116085 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -149,8 +149,9 @@ def get_components_to_install( """ Parse the subworkflow main.nf file to retrieve all imported modules and subworkflows. """ - modules = {} - subworkflows = {} + modules: Dict[str, Dict[str, Optional[str]]] = {} + subworkflows: Dict[str, Dict[str, Optional[str]]] = {} + with open(Path(subworkflow_dir, "main.nf")) as fh: for line in fh: regex = re.compile( @@ -162,16 +163,13 @@ def get_components_to_install( if link.startswith("../../../"): name_split = name.lower().split("_") component_name = "/".join(name_split) - component_dict = { + component_dict: Dict[str, Optional[str]] = { "name": component_name, - "org_path": None, - "git_remote": None, - "branch": None, } modules[component_name] = component_dict elif link.startswith("../"): component_name = name.lower() - component_dict = {"name": component_name, "org_path": None, "git_remote": None, "branch": None} + component_dict = {"name": component_name} subworkflows[component_name] = component_dict if Path(subworkflow_dir, "meta.yml").exists(): @@ -188,7 +186,6 @@ def get_components_to_install( current_comp_dict = subworkflows if component_name in subworkflows else modules component_dict = { - "name": component_name, "org_path": org_path, "git_remote": git_remote, "branch": component[component_name].get("branch", None), From f783d58866f8cbdfdfb795d94d259cbb41096264 Mon Sep 17 00:00:00 2001 From: jvfe Date: Wed, 21 Aug 2024 12:41:24 -0300 Subject: [PATCH 46/55] refact: Use get in modules_json --- nf_core/modules/modules_json.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index b94052c7f0..7c063b572c 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -1255,9 +1255,9 @@ def recreate_dependencies(self, repo, org, subworkflow): assert self.modules_json is not None # mypy for dep_mod in dep_mods: name = dep_mod["name"] - if dep_mod["git_remote"] is not None: - current_repo = dep_mod["git_remote"] - current_org = dep_mod["org_path"] + if dep_mod.get("git_remote") is not None: + current_repo = dep_mod.get("git_remote", repo) + current_org = dep_mod.get("org_path", org) installed_by = self.modules_json["repos"][current_repo]["modules"][current_org][name]["installed_by"] else: installed_by = self.modules_json["repos"][repo]["modules"][org][name]["installed_by"] @@ -1268,9 +1268,9 @@ def recreate_dependencies(self, repo, org, subworkflow): for dep_subwf in dep_subwfs: name = dep_subwf["name"] - if dep_subwf["git_remote"] is not None: - current_repo = dep_subwf["git_remote"] - current_org = dep_subwf["org_path"] + if dep_subwf.get("git_remote") is not None: + current_repo = dep_subwf.get("git_remote", repo) + current_org = dep_subwf.get("org_path", org) installed_by = self.modules_json["repos"][current_repo]["subworkflows"][current_org][name][ "installed_by" ] From eb593089ea395635f178d5487402f3c8595cfa92 Mon Sep 17 00:00:00 2001 From: jvfe Date: Wed, 21 Aug 2024 13:00:04 -0300 Subject: [PATCH 47/55] refact: Use dictionary input in check_up_to_date --- nf_core/modules/modules_json.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 7c063b572c..6afa9b6ec4 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -674,7 +674,7 @@ def check_up_to_date(self): dump_modules_json = True for repo, subworkflows in subworkflows_dict.items(): for org, subworkflow in subworkflows: - self.recreate_dependencies(repo, org, subworkflow) + self.recreate_dependencies(repo, org, {"name": subworkflow}) self.pipeline_components = original_pipeline_components if dump_modules_json: @@ -1249,7 +1249,7 @@ def recreate_dependencies(self, repo, org, subworkflow): i.e., no module or subworkflow has been installed by the user in the meantime """ - sw_name = subworkflow["name"] if isinstance(subworkflow, dict) else subworkflow + sw_name = subworkflow["name"] sw_path = Path(self.subworkflows_dir, org, sw_name) dep_mods, dep_subwfs = get_components_to_install(sw_path) assert self.modules_json is not None # mypy From be4d78f807fbaf5ec0e1aed999220ad542c7f745 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Wed, 21 Aug 2024 22:03:13 +0100 Subject: [PATCH 48/55] Use the power of get to skip if tests --- nf_core/components/components_utils.py | 4 ++-- nf_core/components/install.py | 11 ++++------- nf_core/modules/modules_json.py | 20 ++++++-------------- 3 files changed, 12 insertions(+), 23 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 78c1116085..44c1b7bec7 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -176,7 +176,7 @@ def get_components_to_install( with open(Path(subworkflow_dir, "meta.yml")) as fh: meta = yaml.safe_load(fh) if "components" in meta: - components = meta.get("components") + components = meta["components"] for component in components: if isinstance(component, dict): component_name = list(component.keys())[0].lower() @@ -188,7 +188,7 @@ def get_components_to_install( component_dict = { "org_path": org_path, "git_remote": git_remote, - "branch": component[component_name].get("branch", None), + "branch": component[component_name].get("branch"), } current_comp_dict[component_name].update(component_dict) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index be3d8a40f7..865e681a44 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -51,13 +51,10 @@ def __init__( def install(self, component: str, silent: bool = False) -> bool: if isinstance(component, dict): - if component.get("git_remote") is not None: - remote_url = component.get("git_remote") - branch = component.get("branch") - self.modules_repo = ModulesRepo(remote_url, branch) - else: - self.modules_repo = ModulesRepo(self.current_remote, self.branch) - component = component.get("name") + remote_url = component.get("git_remote", self.current_remote) + branch = component.get("branch", self.branch) + self.modules_repo = ModulesRepo(remote_url, branch) + component = component["name"] if self.repo_type == "modules": log.error(f"You cannot install a {component} in a clone of nf-core/modules") diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 6afa9b6ec4..08e117b1ad 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -1255,12 +1255,9 @@ def recreate_dependencies(self, repo, org, subworkflow): assert self.modules_json is not None # mypy for dep_mod in dep_mods: name = dep_mod["name"] - if dep_mod.get("git_remote") is not None: - current_repo = dep_mod.get("git_remote", repo) - current_org = dep_mod.get("org_path", org) - installed_by = self.modules_json["repos"][current_repo]["modules"][current_org][name]["installed_by"] - else: - installed_by = self.modules_json["repos"][repo]["modules"][org][name]["installed_by"] + current_repo = dep_mod.get("git_remote", repo) + current_org = dep_mod.get("org_path", org) + installed_by = self.modules_json["repos"][current_repo]["modules"][current_org][name]["installed_by"] if installed_by == ["modules"]: self.modules_json["repos"][repo]["modules"][org][dep_mod]["installed_by"] = [] if sw_name not in installed_by: @@ -1268,14 +1265,9 @@ def recreate_dependencies(self, repo, org, subworkflow): for dep_subwf in dep_subwfs: name = dep_subwf["name"] - if dep_subwf.get("git_remote") is not None: - current_repo = dep_subwf.get("git_remote", repo) - current_org = dep_subwf.get("org_path", org) - installed_by = self.modules_json["repos"][current_repo]["subworkflows"][current_org][name][ - "installed_by" - ] - else: - installed_by = self.modules_json["repos"][repo]["subworkflows"][org][name]["installed_by"] + current_repo = dep_subwf.get("git_remote", repo) + current_org = dep_subwf.get("org_path", org) + installed_by = self.modules_json["repos"][current_repo]["subworkflows"][current_org][name]["installed_by"] if installed_by == ["subworkflows"]: self.modules_json["repos"][repo]["subworkflows"][org][dep_subwf]["installed_by"] = [] if sw_name not in installed_by: From b13a40688100eb3782ab01a03e5dfe4dabb66fb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Cavalcante?= Date: Thu, 22 Aug 2024 14:28:46 -0300 Subject: [PATCH 49/55] refact: Raise error if org_path not found --- nf_core/components/components_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 44c1b7bec7..f6bc436de9 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -182,7 +182,10 @@ def get_components_to_install( component_name = list(component.keys())[0].lower() git_remote = component[component_name]["git_remote"] org_path_match = re.search(r"(?:https://|git@)[\w\.]+[:/](.*?)/", git_remote) - org_path = org_path_match.group(1) if org_path_match else None + if org_path_match: + org_path = org_path_match.group(1) + else: + raise UserWarning(f"Organisation path could not be established for {component_name}") current_comp_dict = subworkflows if component_name in subworkflows else modules component_dict = { From e96341e8a3097158d38a222b8559069a7388fcb0 Mon Sep 17 00:00:00 2001 From: jvfe Date: Thu, 22 Aug 2024 14:36:25 -0300 Subject: [PATCH 50/55] style: Run ruff format --- nf_core/components/components_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index f6bc436de9..2190605ab8 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -185,7 +185,7 @@ def get_components_to_install( if org_path_match: org_path = org_path_match.group(1) else: - raise UserWarning(f"Organisation path could not be established for {component_name}") + raise UserWarning(f"Organisation path could not be established for {component_name}") current_comp_dict = subworkflows if component_name in subworkflows else modules component_dict = { From 2b9a573c4b12d6e7961969615283ee0c9aa9bf96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Cavalcante?= Date: Thu, 22 Aug 2024 17:31:43 -0300 Subject: [PATCH 51/55] refact: Change UserWarning message Co-authored-by: Matthieu Muffato --- nf_core/components/components_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 2190605ab8..174002cfff 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -185,7 +185,7 @@ def get_components_to_install( if org_path_match: org_path = org_path_match.group(1) else: - raise UserWarning(f"Organisation path could not be established for {component_name}") + raise UserWarning(f"The organisation path of {component_name} could not be established from '{git_remote}'") current_comp_dict = subworkflows if component_name in subworkflows else modules component_dict = { From e55c74dcc9aaa650ae6160bbf9454401100c203c Mon Sep 17 00:00:00 2001 From: jvfe Date: Thu, 22 Aug 2024 17:36:11 -0300 Subject: [PATCH 52/55] style: Run ruff format --- nf_core/components/components_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index 174002cfff..1b3c1fd182 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -185,7 +185,9 @@ def get_components_to_install( if org_path_match: org_path = org_path_match.group(1) else: - raise UserWarning(f"The organisation path of {component_name} could not be established from '{git_remote}'") + raise UserWarning( + f"The organisation path of {component_name} could not be established from '{git_remote}'" + ) current_comp_dict = subworkflows if component_name in subworkflows else modules component_dict = { From 2491207e1863275d409f7172863df6e27312b3d4 Mon Sep 17 00:00:00 2001 From: jvfe Date: Fri, 6 Sep 2024 14:16:16 -0300 Subject: [PATCH 53/55] refact: Use walrus operator in meta.yml check --- nf_core/components/components_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_utils.py b/nf_core/components/components_utils.py index ddf1301dbd..8a00e758cb 100644 --- a/nf_core/components/components_utils.py +++ b/nf_core/components/components_utils.py @@ -172,8 +172,8 @@ def get_components_to_install( component_dict = {"name": component_name} subworkflows[component_name] = component_dict - if Path(subworkflow_dir, "meta.yml").exists(): - with open(Path(subworkflow_dir, "meta.yml")) as fh: + if (sw_meta := Path(subworkflow_dir, "meta.yml")).exists(): + with open(sw_meta) as fh: meta = yaml.safe_load(fh) if "components" in meta: components = meta["components"] From a870787e04afa12dcd322da174dc9c4a7e3375f7 Mon Sep 17 00:00:00 2001 From: jvfe Date: Fri, 6 Sep 2024 14:27:30 -0300 Subject: [PATCH 54/55] refact: Use a union type for component arg --- nf_core/components/install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index 865e681a44..c9b04b10d6 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -1,7 +1,7 @@ import logging import os from pathlib import Path -from typing import List, Optional, Union +from typing import Dict, List, Optional, Union import questionary from rich import print @@ -49,7 +49,7 @@ def __init__( else: self.installed_by = [self.component_type] - def install(self, component: str, silent: bool = False) -> bool: + def install(self, component: Union[str, Dict[str, str]], silent: bool = False) -> bool: if isinstance(component, dict): remote_url = component.get("git_remote", self.current_remote) branch = component.get("branch", self.branch) From 1a8c3bebd9d6a4dcf9ba54f9b85bc9102f554a1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Cavalcante?= Date: Fri, 13 Sep 2024 10:15:04 -0300 Subject: [PATCH 55/55] docs: Add comment for clarification about the feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/components/install.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/components/install.py b/nf_core/components/install.py index c9b04b10d6..03f419e623 100644 --- a/nf_core/components/install.py +++ b/nf_core/components/install.py @@ -51,6 +51,7 @@ def __init__( def install(self, component: Union[str, Dict[str, str]], silent: bool = False) -> bool: if isinstance(component, dict): + # Override modules_repo when the component to install is a dependency from a subworkflow. remote_url = component.get("git_remote", self.current_remote) branch = component.get("branch", self.branch) self.modules_repo = ModulesRepo(remote_url, branch)