Dev -> Master for 0.5.0 release

.editorconfig

@@ -31,3 +31,7 @@ indent_size = unset
 # ignore python and markdown
 [*.{py,md}]
 indent_style = unset
+
+# ignore multiqc example
+[/assets/multiqc_report.html]
+indent_style = unset

.github/workflows/linting.yml

@@ -11,61 +11,21 @@ on:
     types: [published]
 
 jobs:
-  EditorConfig:
+  pre-commit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
 
-      - uses: actions/setup-node@v3
-
-      - name: Install editorconfig-checker
-        run: npm install -g editorconfig-checker
-
-      - name: Run ECLint check
-        run: editorconfig-checker -exclude README.md $(find .* -type f | grep -v '.git\|.py\|.md\|json\|yml\|yaml\|html\|css\|work\|.nextflow\|build\|nf_core.egg-info\|log.txt\|Makefile')
-
-  Prettier:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-node@v3
-
-      - name: Install Prettier
-        run: npm install -g prettier
-
-      - name: Run Prettier --check
-        run: prettier --check ${GITHUB_WORKSPACE}
-
-  PythonBlack:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Check code lints with Black
-        uses: psf/black@stable
-
-      # If the above check failed, post a comment on the PR explaining the failure
-      - name: Post PR comment
-        if: failure()
-        uses: mshick/add-pr-comment@v1
+      - name: Set up Python 3.12
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5
         with:
-          message: |
-            ## Python linting (`black`) is failing
-
-            To keep the code consistent with lots of contributors, we run automated code consistency checks.
-            To fix this CI test, please run:
-
-            * Install [`black`](https://black.readthedocs.io/en/stable/): `pip install black`
-            * Fix formatting errors in your pipeline: `black .`
-
-            Once you push these changes the test should pass, and you can hide this comment :+1:
+          python-version: "3.12"
 
-            We highly recommend setting up Black in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help!
+      - name: Install pre-commit
+        run: pip install pre-commit
 
-            Thanks again for your contribution!
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          allow-repeats: false
+      - name: Run pre-commit
+        run: pre-commit run --all-files
 
   nf-core:
     runs-on: ubuntu-latest
@@ -84,7 +44,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install nf-core
+          pip install git+https://github.com/nf-core/tools.git@2.14.1
 
       - name: Run nf-core lint
         env:

CHANGELOG.md

@@ -3,6 +3,28 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [[0.5.0](https://github.com/seqeralabs/nf-aggregate/releases/tag/0.5.0)] - 2024-11-12
+
+### Credits
+
+Special thanks to the following for their contributions to the release:
+
+- [Adam Talbot](https://github.com/adamrtalbot)
+- [Esha Joshi](https://github.com/ejseqera)
+- [Jonathan Manning](https://github.com/pinin4fjords)
+- [Maxime Garcia](https://github.com/maxulysse)
+- [Rob Syme](https://github.com/robsyme)
+
+Thank you to everyone else that has contributed by reporting bugs, enhancements or in any other way, shape or form.
+
+### Enhancements & fixes
+
+- [PR #61](https://github.com/seqeralabs/nf-aggregate/pull/61) - Remove dependency on external library/grape
+- [PR #63](https://github.com/seqeralabs/nf-aggregate/pull/63) - Add `maxForks` setting for Seqera CLI to overcome API issues
+- [PR #65](https://github.com/seqeralabs/nf-aggregate/pull/65) - Replace eclint GHA by pre-commit
+- [PR #67](https://github.com/seqeralabs/nf-aggregate/pull/67) - Update tests to use a non-fusion run from Seqera Cloud community/showcase
+- [PR #69](https://github.com/seqeralabs/nf-aggregate/pull/69) - Fix parsing of extra args to tw CLI
+
 ## [[0.4.0](https://github.com/seqeralabs/nf-aggregate/releases/tag/0.4.0)] - 2024-07-26
 
 ### Credits
@@ -15,9 +37,9 @@ Thank you to everyone else that has contributed by reporting bugs, enhancements
 
 ### Enhancements & fixes
 
-[PR #52](https://github.com/seqeralabs/nf-aggregate/pull/52) - Organise results folder structure by pipeline
-[PR #53](https://github.com/seqeralabs/nf-aggregate/pull/53) - Throw exception and terminate workflow in case config can't be read
-[PR #57](https://github.com/seqeralabs/nf-aggregate/pull/57) - Check if fusion is enabled via the Platform API
+- [PR #52](https://github.com/seqeralabs/nf-aggregate/pull/52) - Organise results folder structure by pipeline
+- [PR #53](https://github.com/seqeralabs/nf-aggregate/pull/53) - Throw exception and terminate workflow in case config can't be read
+- [PR #57](https://github.com/seqeralabs/nf-aggregate/pull/57) - Check if fusion is enabled via the Platform API
 
 ## [[0.3.0](https://github.com/seqeralabs/nf-aggregate/releases/tag/0.3.0)] - 2024-07-01
 

modules/local/plot_run_gantt/nextflow.config

@@ -6,4 +6,4 @@ process {
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
-}
+}

modules/local/seqera_runs_dump/functions.nf

@@ -1,78 +1,65 @@
-@Grab('com.github.groovy-wslite:groovy-wslite:1.1.2;transitive=false')
-import wslite.rest.RESTClient
-import groovy.json.JsonSlurper
-import nextflow.exception.ProcessException
-import groovy.json.JsonBuilder
 
-// Set system properties for custom Java trustStore
-def setTrustStore(trustStorePath, trustStorePassword) {
-    System.setProperty("javax.net.ssl.trustStore", trustStorePath)
-    if (trustStorePassword) {
-        System.setProperty("javax.net.ssl.trustStorePassword", trustStorePassword)
+Long getWorkspaceId(orgName, workspaceName, api_endpoint, authHeader) {
+    Map response
+    try {
+        def responseString = new URL("${api_endpoint}/orgs").getText(requestProperties: authHeader)
+        response = new groovy.json.JsonSlurper().parseText(responseString)
+    } catch (Exception e) {
+        log.warn "Could not fetch organization ${orgName} from API endpoint ${api_endpoint}"
+        throw new nextflow.exception.ProcessException("Failed to get organization details for ${orgName} in workspace ${workspaceName}", e)
     }
-}
 
-Long getWorkspaceId(orgName, workspaceName, client, authHeader) {
-    def orgResponse = client.get(path: '/orgs', headers: authHeader)
-    if (orgResponse.statusCode == 200) {
-        def orgMap = orgResponse.json?.organizations.collectEntries { org -> [org.name, org.orgId] }
-        def orgId = orgMap.get(orgName)
-        if(!orgId) log.warn "Could not find organization '${orgName}'"
+    def orgId = response
+        ?.organizations
+        ?.collectEntries { org -> [org.name, org.orgId] }
+        ?.get(orgName)
+    if(!orgId) log.warn "Could not find organization '${orgName}'"
 
-        // GET the workspaces in this org
-        def workspaceReponse = client.get(path: "/orgs/${orgId}/workspaces", headers: authHeader)
-        if (workspaceReponse.statusCode == 200) {
-            def workspaceMap = workspaceReponse.json?.workspaces.collectEntries { ws -> [ws.name, ws.id]}
-            return workspaceMap?.get(workspaceName)
-        } else {
-            log.error "Failed to fetch workspaces for orgId: ${orgId}, statusCode: ${workspaceResponse.statusCode}"
-        }
+    try {
+        def workspaceReponse = new URL("${api_endpoint}/orgs/${orgId}/workspaces").getText(requestProperties: authHeader)
+        def workspaceMap = new groovy.json.JsonSlurper()
+            .parseText(workspaceReponse)
+            ?.workspaces
+            ?.collectEntries { ws -> [ws.name, ws.id]}
+        return workspaceMap?.get(workspaceName)
+    } catch (Exception e) {
+        log.error "Failed to fetch workspaces for orgId: ${orgId}"
+        return null
     }
-    return null
 }
 
 Map getRunMetadata(meta, log, api_endpoint, trustStorePath, trustStorePassword) {
     def runId = meta.id
     def (orgName, workspaceName) = meta.workspace.tokenize("/")
 
-    if (trustStorePath) {
-        log.info "Setting custom truststore: ${trustStorePath}"
-        setTrustStore(trustStorePath, trustStorePassword)
-    }
-
-    def client = new RESTClient(api_endpoint)
     def token = System.getenv("TOWER_ACCESS_TOKEN")
     def authHeader = ["Authorization": "Bearer ${token}"]
+    def workspaceId = getWorkspaceId(orgName, workspaceName, api_endpoint, authHeader)
+    def endpointUrl = "${api_endpoint}/workflow/${runId}?workspaceId=${workspaceId}"
+    if(!workspaceId) {
+        log.error "Failed to get workspaceId for ${orgName}/${workspaceName}"
+        return [:]
+    }
 
     try {
-        def workspaceId = getWorkspaceId(orgName, workspaceName, client, authHeader)
-        if (workspaceId) {
-            def workflowResponse = client.get(path: "/workflow/${runId}", query: ["workspaceId":workspaceId], headers: authHeader)
-            if (workflowResponse.statusCode == 200) {
-                def metaMap = workflowResponse?.json?.workflow?.subMap("runName", "workDir", "projectName")
-                def configText = new JsonBuilder(workflowResponse?.json?.workflow?.configText)
-                def pattern = /fusion\s*\{\\n\s*enabled\s*=\s*true/
-                def matcher = configText.toPrettyString() =~ pattern
-                metaMap.fusion = matcher.find()
+        def workflowResponseString = new URL(endpointUrl).getText(requestProperties: authHeader)
+        def workflowResponse = new groovy.json.JsonSlurper().parseText(workflowResponseString)
+        def metaMap = workflowResponse
+            ?.workflow
+            ?.subMap("runName", "workDir", "projectName")
+        def configText = new groovy.json.JsonBuilder(workflowResponse?.workflow?.configText)
+        def pattern = /fusion\s*\{\\n\s*enabled\s*=\s*true/
+        def matcher = configText.toPrettyString() =~ pattern
+        metaMap.fusion = matcher.find()
 
-                return metaMap ?: [:]
-            }
-        }
-    } catch (wslite.rest.RESTClientException ex) {
-        log.warn """
-        Could not get workflow details for workflow ${runId} in workspace ${meta.workspace}:
-            ↳ Status code ${ex.response?.statusCode} returned from request to ${ex.request?.url} (authentication headers excluded)
-        """.stripIndent()
-        log.error "Exception: ${ex.message}", ex
-        throw new ProcessException("Failed to get workflow details for workflow ${runId} in workspace ${meta.workspace}", ex)
+        return metaMap ?: [:]
     } catch (Exception ex) {
         log.warn """
-        An error occurred while getting workflow details for workflow ${runId} in workspace ${meta.workspace}:
-            ↳ ${ex.message}
+        Could not get workflow details for workflow ${runId} in workspace ${meta.workspace}:
+            ↳ From request to ${endpointUrl}
         """.stripIndent()
         log.error "Exception: ${ex.message}", ex
-        throw new ProcessException("Failed to get workflow details for workflow ${runId} in workspace ${meta.workspace}", ex)
-
+        throw new nextflow.exception.ProcessException("Failed to get workflow details for workflow ${runId} in workspace ${meta.workspace}", ex)
     }
     return [:]
 }

modules/local/seqera_runs_dump/main.nf

@@ -20,7 +20,7 @@ process SEQERA_RUNS_DUMP {
     def args2 = task.ext.args2 ?: ''
     prefix = task.ext.prefix ?: "${meta.id}"
     metaOut = meta + getRunMetadata(meta, log, api_endpoint, java_truststore_path, java_truststore_password)
-    fusion = metaOut.fusion ? '--add-fusion-logs' : ''
+    def fusion = metaOut.fusion ? '--add-fusion-logs' : ''
     javaTrustStore = java_truststore_path ? "-Djavax.net.ssl.trustStore=${java_truststore_path}" : ''
     javaTrustStorePassword = java_truststore_password ? "-Djavax.net.ssl.trustStorePassword=${java_truststore_password}" : ''
     """

modules/local/seqera_runs_dump/nextflow.config

@@ -1,6 +1,7 @@
 process {
     withName: 'SEQERA_RUNS_DUMP' {
-        ext.args = { params.seqera_cli_extra_args ? params.seqera_cli_extra_args.split("\\s(?=--)") : '' }
+        maxForks = 5
+        ext.args = { params.seqera_cli_extra_args ? params.seqera_cli_extra_args.split(/[\s,]+(?=--)/).join(' ') : '' }
         ext.args2 = { params.skip_run_gantt ? '' : '--add-task-logs' }
         containerOptions = { params.java_truststore_path ? "--volume ${params.java_truststore_path}:${params.java_truststore_path}" : '' }
         publishDir = [

modules/local/seqera_runs_dump/tests/main.nf.test.snap

@@ -2,7 +2,7 @@
     "Should run without failures": {
         "content": [
             [
-                "service-info.json:md5,72586e82ad2064d1a90008c7956d80d8",
+                "service-info.json:md5,75a5b8cded8f0ce9aab2125c0ee89f63",
                 "workflow-load.json:md5,4f02d5a24ab89aa648cd4346785c8f2c",
                 "workflow-metadata.json:md5,b37b4faeddf283a2c44cbe4000e4ab6e",
                 "workflow-metrics.json:md5,13a5b5d7447fad4a8baa053d1abf85e5",
@@ -12,9 +12,9 @@
             null
         ],
         "meta": {
-            "nf-test": "0.8.4",
-            "nextflow": "24.04.2"
+            "nf-test": "0.9.2",
+            "nextflow": "23.10.1"
         },
-        "timestamp": "2024-07-25T10:21:06.447483"
+        "timestamp": "2024-11-11T11:53:03.835699"
     }
-}
+}

nextflow.config

@@ -222,6 +222,6 @@ manifest {
     mainScript      = 'main.nf'
     nextflowVersion = '!>=23.10.0'
     defaultBranch   = 'main'
-    version         = '0.4.0'
+    version         = '0.5.0'
     doi             = ''
 }

subworkflows/local/utils_nf_aggregate/main.nf

@@ -78,7 +78,7 @@ def getWorkflowPublishDir(json_file, outdir) {
         def workdir = getWorkflowWorkDir(json_file)
         if (workdir.startsWith('s3://')) {
             path = Paths.get(workdir, path)
-        }  
+        }
     }
     return path
 }
@@ -99,8 +99,8 @@ def getProcessVersions(yaml_file) {
 def getWorkflowVersions() {
     return """
     'Workflow':
-      "Nextflow": "$workflow.nextflow.version"
-      "$workflow.manifest.name": "$workflow.manifest.version"
+        "Nextflow": "$workflow.nextflow.version"
+        "$workflow.manifest.name": "$workflow.manifest.version"
     """.stripIndent().trim()
 }
 

subworkflows/local/utils_nf_aggregate/tests/main.nf.test

@@ -6,7 +6,7 @@ nextflow_workflow {
 
     tag "subworkflows"
     tag "subworkflows_local_utils_nf_aggregate"
-    
+
     stage {
         symlink "nextflow_schema.json"
         symlink "workflows/nf_aggregate/assets/test_run_ids.csv"
@@ -28,10 +28,8 @@ nextflow_workflow {
                 { assert workflow.success },
                 { assert snapshot(
                     workflow.out.ids
-                 ).match() }
+                ).match() }
             )
         }
-
     }
-
 }

subworkflows/local/utils_nf_aggregate/tests/main.nf.test.snap

@@ -2,10 +2,6 @@
     "Should run without failures": {
         "content": [
             [
-                {
-                    "id": "1oXDHFHywk3wR2",
-                    "workspace": "scidev/testing"
-                },
                 {
                     "id": "2lXd1j7OwZVfxh",
                     "workspace": "community/showcase"
@@ -14,6 +10,10 @@
                     "id": "38QXz4OfQDpwOV",
                     "workspace": "community/showcase"
                 },
+                {
+                    "id": "3iFMo0NtH1Byvy",
+                    "workspace": "community/showcase"
+                },
                 {
                     "id": "4Bi5xBK6E2Nbhj",
                     "workspace": "community/showcase"
@@ -25,9 +25,9 @@
             ]
         ],
         "meta": {
-            "nf-test": "0.8.4",
-            "nextflow": "23.10.1"
+            "nf-test": "0.9.2",
+            "nextflow": "24.04.2"
         },
-        "timestamp": "2024-05-07T10:41:28.603749237"
+        "timestamp": "2024-11-11T11:32:52.476285"
     }
 }

workflows/nf_aggregate/assets/test_run_ids.csv

@@ -3,4 +3,4 @@ id,workspace
 4LWT4uaXDaGcDY,community/showcase
 38QXz4OfQDpwOV,community/showcase
 2lXd1j7OwZVfxh,community/showcase
-1oXDHFHywk3wR2,scidev/testing
+3iFMo0NtH1Byvy,community/showcase

workflows/nf_aggregate/nextflow.config

@@ -1,3 +1,3 @@
 includeConfig '../../modules/local/seqera_runs_dump/nextflow.config'
 includeConfig '../../modules/local/plot_run_gantt/nextflow.config'
-includeConfig '../../modules/nf-core/multiqc/nextflow.config'
+includeConfig '../../modules/nf-core/multiqc/nextflow.config'