Merge remote-tracking branch 'origin/main'

.github/workflows/test.yaml

@@ -13,6 +13,17 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v3
 
+    - name: Free up disk space
+      uses: jlumbroso/free-disk-space@main
+      with:
+        tool-cache: false
+        android: true
+        dotnet: true
+        haskell: true
+        large-packages: false
+        docker-images: true
+        swap-storage: false
+
     - uses: actions/setup-java@v3
       with:
         distribution: 'zulu'

dependencies/py-environment.yml

@@ -6,16 +6,14 @@ dependencies:
   - python==3.11
   - pyspark==3.3.2
   - openjdk
+  - jsonapi-client==0.9.9
+  - pandas==2.1.0
+  - ipykernel==6.25.1
+  - matplotlib==3.7.2
+  - seaborn==0.12.2
+  - plotly==5.16.1
+  - biopython==1.81
+  - pyarrow==13.0.0
   - pip
   - pip:
-    - jsonapi-client==0.9.9
-    - pandas==1.5.3
-    - numpy==1.24.2
-    - ipykernel==6.21.3
-    - matplotlib==3.7.1
-    - seaborn==0.12.2
-    - plotly==5.13.1
-    - sourmash==4.1.2
-    - biopython==1.81
-    - pyarrow==11.0.0
-
+    - sourmash==4.1.2

docker/Dockerfile

@@ -1,35 +1,70 @@
-FROM jupyter/minimal-notebook:lab-3.6.3@sha256:1443243e00caa7db61de7faaad626a173c4a51d8f15e48e07c6b111de73d0f38
-LABEL maintainer="EMBL-EBI Microbiome Informatics (MGnify) Team <metagenomics-help@ebi.ac.uk>"
-USER root
-ENV CHOWN_HOME_OPTS='-R'
-ENV CHOWN_HOME='yes'
+FROM continuumio/miniconda3 as build
+WORKDIR /work
 
-# Install Python/R dependencies
-RUN mamba install -y nb_conda_kernels
 RUN conda config --system --prepend channels bioconda
 COPY dependencies/r-environment.yml /tmp/r-environment.yml
-RUN mamba env create -f /tmp/r-environment.yml
+RUN conda env create -f /tmp/r-environment.yml
+
 COPY dependencies/py-environment.yml /tmp/py-environment.yml
-RUN mamba env create -f /tmp/py-environment.yml
-COPY dependencies/dependencies.R /tmp/dependencies.R
+RUN conda env create -f /tmp/py-environment.yml
+RUN conda clean -afy
 
+COPY dependencies/dependencies.R /tmp/dependencies.R
 SHELL ["conda", "run", "-n", "mgnify-r-env", "/bin/bash", "-c"]
 RUN Rscript /tmp/dependencies.R
+SHELL ["/bin/bash", "-c"]
+
+RUN conda install -c conda-forge conda-pack && conda clean --all -f -y
+
+RUN conda-pack -n mgnify-r-env \
+    -o /tmp/mgnify-r-env.tar \
+    --exclude *.js.map \
+    --exclude *.pyc \
+    --exclude *.a && \
+    mkdir -p /tmp/envs/mgnify-r-env && cd /tmp/envs/mgnify-r-env && tar xf /tmp/mgnify-r-env.tar && \
+    rm /tmp/mgnify-r-env.tar && \
+    rm -rf /opt/conda/envs/mgnify-r-env && \
+    mv /tmp/envs/mgnify-r-env /opt/conda/envs/mgnify-r-env && \
+    /opt/conda/envs/mgnify-r-env/bin/conda-unpack
+
+RUN conda-pack -n mgnify-py-env \
+    -o /tmp/mgnify-py-env.tar \
+    --exclude *.js.map \
+    --exclude *.pyc \
+    --exclude *.a && \
+    mkdir -p /tmp/envs/mgnify-py-env && cd /tmp/envs/mgnify-py-env && tar xf /tmp/mgnify-py-env.tar && \
+    rm /tmp/mgnify-py-env.tar && \
+    rm -rf /opt/conda/envs/mgnify-py-env && \
+    mv /tmp/envs/mgnify-py-env /opt/conda/envs/ && \
+    /opt/conda/envs/mgnify-py-env/bin/conda-unpack
+
+# ------------------- #
+
+FROM jupyter/base-notebook:lab-3.6.3@sha256:8a15ee10ec8be7c760c83f8fb87445530bf8a92dfddd99ec8dab240f388fdfdf as runtime
+LABEL maintainer="EMBL-EBI Microbiome Informatics (MGnify) Team <metagenomics-help@ebi.ac.uk>"
+USER root
+ENV CHOWN_HOME_OPTS='-R'
+ENV CHOWN_HOME='yes'
+
+WORKDIR /work
+
+COPY --chown=root:100 --from=build /opt/conda/envs /opt/conda/envs
+
+# Install Python/R dependencies
+RUN mamba install -y conda-forge::nb_conda_kernels --freeze-installed && mamba clean --all -f -y
+
 # Place / check / populate MGnifyR cache for example used in studies
 # Zipped cache should be up to date in repo, but run populate to be sure nothing is missing
 COPY dependencies/mgnify-cache.tgz /tmp/mgnify-cache.tgz
 RUN tar -xzf /tmp/mgnify-cache.tgz -C /
-COPY dependencies/populate-mgnifyr-cache.R /tmp/populate-mgnifyr-cache.R
-RUN Rscript /tmp/populate-mgnifyr-cache.R
-SHELL ["/bin/bash", "-c"]
 
 # Install JupyterLab extension to handle query parameter > env vars in ShinyProxy
 COPY jlab_query_params /tmp/jlab_query_params
 RUN pip install /tmp/jlab_query_params
 
 # Install Jupyter Lab extension providing MGnify-specific help 
-#COPY mgnify_jupyter_lab_ui /tmp/mgnify_jupyter_lab_ui
-#RUN pip install /tmp/mgnify_jupyter_lab_ui
+COPY mgnify_jupyter_lab_ui /tmp/mgnify_jupyter_lab_ui
+RUN pip install /tmp/mgnify_jupyter_lab_ui
 
 # Clean yarn cache else chown'ing home is very slow on container start
 RUN rm -rf /home/jovyan/.yarn
@@ -43,3 +78,5 @@ COPY jupyter_config/custom.js /home/jovyan/.jupyter/custom/custom.js
 COPY jupyter_config/jupyter_config.json /home/jovyan/.jupyter/jupyter_config.json
 COPY src/notebooks /home/jovyan/mgnify-examples
 RUN jupyter labextension disable "@jupyterlab/apputils-extension:announcements"
+
+WORKDIR /home/jovyan