Merge pull request #1 from papaemmelab/add-test

✅ Add ci with nf-test
papaemmelab · Jan 23, 2024 · b31b329 · b31b329
2 parents 3eb80ad + ff7367e
commit b31b329
Show file tree

Hide file tree

Showing 34 changed files with 3,801 additions and 58,101 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,4 +1,4 @@
-name: nf-all-classifiers CI
+name: nf-purple CI
 
 on:
   push:
@@ -11,19 +11,21 @@ jobs:
     runs-on: ubuntu-latest
     name: Job to run the test workflow
     steps:
-      - uses: actions/checkout@v1
-      - name: Install Nextflow
-        env:
-          CAPSULE_LOG: none
-        run: |
-          wget -qO- get.nextflow.io | bash
-          sudo mv nextflow /usr/local/bin/
-
+      - uses: actions/checkout@v3
+      - uses: nf-core/setup-nextflow@v1
       - name: Install nf-test
         run: |
           wget -qO- https://code.askimed.com/install/nf-test | bash
           sudo mv nf-test /usr/local/bin/
-      - name: Run test data
-        run:
-          sh tests/run_test.sh
-
+      - name: Pull Docker image and cache
+        run: |
+          docker pull papaemmelab/purple:v0.1.0
+      - name: Run unit tests of each process for Amber, Cobalt, Purple
+        run: |
+          nf-test test tests/main.runamber.nf.test
+          nf-test test tests/main.runcobalt.nf.test
+          nf-test test tests/main.runpurple.nf.test
+      - name: Run pipeline end-to-end test
+        run: |
+          nf-test test tests/main.nf.test
+      
diff --git a/.gitignore b/.gitignore
@@ -5,6 +5,7 @@ capsule
 framework
 pipeline_*
 .nextflow*
+.nf-test*
 tmp
 
 # Tests

diff --git a/Dockerfile b/Dockerfile
@@ -1,16 +1,26 @@
-FROM nextflow/nextflow
+FROM papaemmelab/docker-hmftools:v1.0.0
 
 # Install dependencies
-RUN yum install -y tar
+RUN apt-get update && apt-get install -y tar curl python3
 
-# Install https://github.com/askimed/nf-test
-RUN mkdir -p /opt/bin && \
-    cd /opt/bin && \
-    curl -fsSL https://code.askimed.com/install/nf-test | bash && \
-    export PATH=/opt/bin:$PATH && \
-    nf-test version
+# Install Google Cloud SDK
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH $PATH:/root/google-cloud-sdk/bin
+
+# Set version as an environment variable
+ENV GENOME_VERSION=37
+ENV HMFTOOLS_VERSION=5.33
+ENV HMFTOOLS_VERSION_UNDERSCORE 5_33
+ENV REF_DIR=hmf_dna_pipeline_resources.${GENOME_VERSION}_v${HMFTOOLS_VERSION}
+
+# Download the file using gsutil
+RUN \
+    mkdir -p /data && \
+    gsutil cp gs://hmf-public/HMFtools-Resources/dna_pipeline/v${HMFTOOLS_VERSION_UNDERSCORE}/${GENOME_VERSION}/${REF_DIR}.tar.gz /data/ && \
+    tar -xzf /data/${REF_DIR}.tar.gz -C /data && \
+    rm /data/${REF_DIR}.tar.gz
 
 COPY . /app
 WORKDIR /app
 
-ENTRYPOINT ["nextflow", "-version"]
+ENTRYPOINT ["bash", "-c"]
diff --git a/README.md b/README.md
@@ -1,10 +1,11 @@
+# 🟣 nf-purple
+
 [![nf-purple CI](https://github.com/papaemmelab/nf-purple/actions/workflows/ci.yml/badge.svg)](https://github.com/papaemmelab/nf-purple/actions/workflows/ci.yml)
 [![nf-test](https://img.shields.io/badge/tested_with-nf--test-337ab7.svg)](https://github.com/askimed/nf-test)
-# nf-purple
 
-Nextflow Pipeline to run [Purple](https://github.com/hartwigmedical/hmftools/blob/master/purple/README.md#tumor-only-mode), uses [Amber](https://github.com/hartwigmedical/hmftools/tree/master/amber#tumor-only-mode) and [Cobalt](https://github.com/hartwigmedical/hmftools/tree/master/cobalt) from HMFTools suite, of the Hartwig Foundation.
+Nextflow Pipeline to run [Purple](https://github.com/hartwigmedical/hmftools/blob/master/purple/README.md#tumor-only-mode) in *Tumor-Only* mode, uses [Amber](https://github.com/hartwigmedical/hmftools/tree/master/amber#tumor-only-mode) and [Cobalt](https://github.com/hartwigmedical/hmftools/tree/master/cobalt#tumor-only-mode) from HMFTools suite, of the Hartwig Foundation.
 
-## Run Pipeline
+## 🚀 Run Pipeline
 
 You need Nextflow installed.
 
@@ -18,9 +19,39 @@ nextflow papaemmelab/nf-purple \
     ...refargs
 ```
 
-### Tests
+## 🧬  Get Reference Data
+
+Downloaded from [Purple Ref Data](https://console.cloud.google.com/storage/browser/hmf-public/HMFtools-Resources/dna_pipeline) for genome version 37.
+More information on their [docs](https://github.com/hartwigmedical/hmftools/blob/master/purple/README.md).
+
+## 📒 Tools Info
+
+### Purple
+
+>[Purple](https://github.com/hartwigmedical/hmftools/blob/master/purple/README.md) is a **pur**ity **pl**oidy **e**stimator for whole genome sequenced (WGS) data. It combines B-allele frequency (BAF) from AMBER, read depth ratios from COBALT, somatic variants and structural variants to estimate the purity and copy number profile of a tumor sample. PURPLE supports both grch 37 and 38 reference assemblies.
+
+### Amber
+
+>[Amber](https://github.com/hartwigmedical/hmftools/tree/master/amber) is designed to generate a tumor BAF file for use in PURPLE from a provided VCF of likely heterozygous SNP sites.
+
+### Cobalt
+
+>[Cobalt](https://github.com/hartwigmedical/hmftools/tree/master/cobalt). **C**ount **ba**m **l**ines determines the read depth ratios of the supplied **t**umor and reference genomes.
+
+## 🕵🏻‍♂️ Tests
 
 ```bash
 sh tests/run_test.sh
 ```
-# nf-purple
+
+### Docker
+
+[Purple Docker image](https://hub.docker.com/r/papaemmelab/purple) was built for several platforms using [docker buildx](https://docs.docker.com/buildx/working-with-buildx/).
+
+```bash
+# Create a new builder instance
+docker buildx create --name papaemmelab-builder --use
+
+# Build the image
+docker buildx build --platform linux/amd64,linux/arm64 -t papaemmelab/purple:v0.1.0 . --push
+```
diff --git a/docker/Dockerfile.allsorts b/docker/Dockerfile.allsorts
diff --git a/docker/Dockerfile.allspice.allcatchr b/docker/Dockerfile.allspice.allcatchr
diff --git a/main.nf b/main.nf
@@ -1,13 +1,13 @@
-params.cores = 16
+params.cores = 4
 
 // Params Defaults in juno
 params.refGenome = "/work/isabl/ref/homo_sapiens/GRCh37d5/gr37.fasta"
-params.circos = "/opt/circos-0.69-2/bin/circos"
-params.loci = "ref/copy_number/GermlineHetPon.37.vcf.gz"
-params.gcProfile = "ref/copy_number/GC_profile.1000bp.37.cnp"
-params.ensemblDataDir = "ref/common/ensembl_data"
 params.genomeVersion = "V37"
-params.diploidRegions = "copy_number/DiploidRegions.37.bed.gz"
+params.circos = "/opt/circos-0.69-2/bin/circos"
+params.loci = "/data/copy_number/GermlineHetPon.37.vcf.gz"
+params.gcProfile = "/data/copy_number/GC_profile.1000bp.37.cnp"
+params.ensemblDataDir = "/data/common/ensembl_data"
+params.diploidRegions = "/data/copy_number/DiploidRegions.37.bed.gz"
 
 
 log.info """\
@@ -36,24 +36,24 @@ process runAmber {
     time '1h'
 
     input:
-    path tumor
+    val tumor
     path tumorBam
 
     output:
-    path "${tumor}.amber.baf.tsv.gz"
-    path "${tumor}.amber.baf.pcf"
-    path "${tumor}.amber.qc"
+    path "${tumor}.amber.baf.tsv.gz", emit: amber_baf_tsv
+    path "${tumor}.amber.baf.pcf", emit: amber_baf_pcf
+    path "${tumor}.amber.qc", emit: amber_qc
 
     script:
     """
-    hmftools amber \
+    amber \
         -tumor ${tumor} \
         -tumor_bam ${tumorBam} \
         -output_dir \$PWD \
         -threads ${params.cores} \
         -loci ${params.loci} \
         -ref_genome_version ${params.genomeVersion}
-    """
+    """.stripIndent()
 }
 
 process runCobalt {
@@ -64,23 +64,23 @@ process runCobalt {
     time '1h'
 
     input:
-    path tumor
+    val tumor
     path tumorBam
 
     output:
-    path "${tumor}.cobalt.ratio.tsv.gz"
-    path "${tumor}.cobalt.ratio.pcf"
+    path "${tumor}.cobalt.ratio.tsv.gz", emit: cobalt_ratio_tsv
+    path "${tumor}.cobalt.ratio.pcf", emit: cobalt_ratio_pcf
 
     script:
     """
-    hmftools cobalt \
+    cobalt \
         -tumor ${tumor} \
         -tumor_bam ${tumorBam} \
         -output_dir \$PWD \
         -threads ${params.cores} \
         -gc_profile ${params.gcProfile} \
         -tumor_only_diploid_bed ${params.diploidRegions}
-    """
+    """.stripIndent()
 }
 
 process runPurple {
@@ -91,45 +91,45 @@ process runPurple {
     time '1h'
 
     input:
-    path tumor
+    val tumor
     path amber_baf_tsv
     path amber_baf_pcf
     path amber_qc
     path cobalt_ratio_tsv
     path cobalt_ratio_pcf
 
     output:
-    path "${tumor}.purple.purity.tsv"
-    path "${tumor}.purple.qc"
-    path "${tumor}.purple.purity.range.tsv"
-    path "${tumor}.purple.cnv.somatic.tsv"
-    path "${tumor}.purple.cnv.gene.tsv"
-    path "${tumor}.purple.segment.tsv"
-    path "${tumor}.purple.somatic.clonality.tsv"
-    path "plot/${tumor}.segment.png"
-    path "plot/${tumor}.copynumber.png"
-    path "plot/${tumor}.circos.png"
-    path "plot/${tumor}.map.png"
-    path "plot/${tumor}.input.png"
-    path "plot/${tumor}.purity.range.png"
+    path "${tumor}.purple.purity.tsv", emit: purple_purity_tsv
+    path "${tumor}.purple.qc", emit: purple_qc
+    path "${tumor}.purple.purity.range.tsv", emit: purple_purity_range_tsv
+    path "${tumor}.purple.cnv.somatic.tsv", emit: purple_cnv_somatic_tsv
+    path "${tumor}.purple.cnv.gene.tsv", emit: purple_cnv_gene_tsv
+    path "${tumor}.purple.segment.tsv", emit: purple_segment_tsv
+    path "${tumor}.purple.somatic.clonality.tsv", emit: purple_somatic_clonality_tsv
+    path "plot/${tumor}.segment.png", emit: purple_segment_png
+    path "plot/${tumor}.copynumber.png", emit: purple_copynumber_png
+    path "plot/${tumor}.circos.png", emit: purple_circos_png
+    path "plot/${tumor}.map.png", emit: purple_map_png
+    path "plot/${tumor}.input.png", emit: purple_input_png
+    path "plot/${tumor}.purity.range.png", emit: purple_purity_range_png
 
     script:
     """
-    hmftools purple \
-    -tumor ${tumor} \
-    -amber ${params.outdir}/amber \
-    -cobalt ${params.outdir}/cobalt \
-    -output_dir \$PWD \
-    -gc_profile ${params.gcProfile} \
-    -ref_genome ${params.refGenome} \
-    -ref_genome_version ${params.genomeVersion} \
-    -ensembl_data_dir ${params.ensemblDataDir} \
-    -circos ${params.circos}
-    """
+    purple \
+        -tumor ${tumor} \
+        -amber ${params.outdir}/amber \
+        -cobalt ${params.outdir}/cobalt \
+        -output_dir \$PWD \
+        -gc_profile ${params.gcProfile} \
+        -ref_genome ${params.refGenome} \
+        -ref_genome_version ${params.genomeVersion} \
+        -ensembl_data_dir ${params.ensemblDataDir} \
+        -circos ${params.circos}
+    """.stripIndent()
 }
 
 workflow {
-    tumor = Channel.fromPath(params.tumor)
+    tumor = Channel.value(params.tumor)
     tumorBam = Channel.fromPath(params.tumorBam)
 
     runAmber(tumor, tumorBam)

diff --git a/nextflow.config b/nextflow.config
@@ -20,28 +20,24 @@ report {
 
 // Profiles
 profiles {
-
     standard {
         process.executor = 'local'
     }
-
     hpc {
         singularity.enabled = true
         singularity.autoMounts = true
         process {
             executor = 'lsf'
         }
     }
-
     cloud {
         docker {
           enabled = true
           runOptions = '--entrypoint ""'
         }
-
         process {
             executor = 'local'
-            container = '/juno/work/isabl/local/hmftools/hmftools.simg'
+            container = 'papaemmelab/purple:v0.1.0'
         }
     }
 }

diff --git a/nf-test.config b/nf-test.config
@@ -0,0 +1,8 @@
+config {
+
+    testsDir "tests"
+    workDir ".nf-test"
+    configFile "tests/nextflow.config"
+    profile "cloud"
+
+}