test build of itwinai - vre image

.github/workflows/env-build.yaml

@@ -3,7 +3,7 @@ name: Docker automatic build and publish
 on:
   push:
     branches: 
-      - main
+      - itwinai_integration
 
 env:
   REGISTRY: ghcr.io

vre-singleuser-itwinai/Dockerfile

@@ -0,0 +1,87 @@
+FROM nvcr.io/nvidia/pytorch:23.09-py3 
+LABEL author="E. Garcia, M. Bunino"
+LABEL maintainer="VRE Team @ CERN 23/24 - E. Garcia, G. Guerrieri"
+
+USER root
+
+RUN apt-get update -y
+RUN apt-get install -y libglib2.0-dev libgfal2-dev python3-dev libboost-all-dev
+
+RUN python -m pip install --upgrade pip
+RUN python -m pip install 'jupyterhub==1.5.0'
+RUN npm install -g configurable-http-proxy
+RUN python -m pip install \
+    'jupyterlab' \
+    'notebook<7' \
+    'nbclassic' \
+    jupyter server --generate-config \
+    asyncssh
+
+RUN python -m pip install rucio-jupyterlab==0.10.0 \
+    && jupyter serverextension enable --py rucio_jupyterlab --sys-prefix 
+
+RUN apt update -y \
+    && apt install -y build-essential curl voms-clients-java software-properties-common \
+    && apt clean -y \
+    && rm /opt/conda/bin/voms-proxy-init \
+    && ln -s /usr/bin/voms-proxy-init /opt/conda/bin/voms-proxy-init
+
+RUN wget -q -O - https://dist.eugridpma.info/distribution/igtf/current/GPG-KEY-EUGridPMA-RPM-3 | apt-key add -
+
+RUN apt update \
+    && add-apt-repository 'deb http://repository.egi.eu/sw/production/cas/1/current egi-igtf core' \
+    && apt -y install ca-policy-egi-core
+
+RUN mkdir -p /etc/vomses \
+    && wget https://indigo-iam.github.io/escape-docs/voms-config/voms-escape.cloud.cnaf.infn.it.vomses -O /etc/vomses/voms-escape.cloud.cnaf.infn.it.vomses \
+    && mkdir -p /etc/grid-security/vomsdir/escape \
+    && wget https://indigo-iam.github.io/escape-docs/voms-config/voms-escape.cloud.cnaf.infn.it.lsc -O /etc/grid-security/vomsdir/escape/voms-escape.cloud.cnaf.infn.it.lsc
+
+# Setup merged CERN CA file on Ubuntu based images.
+# This file is contained in the `CERN-bundle.pem` file downloaded using 
+RUN mkdir /certs \
+    && touch /certs/rucio_ca.pem \
+    && curl -fsSL 'https://cafiles.cern.ch/cafiles/certificates/CERN%20Root%20Certification%20Authority%202.crt' | openssl x509 -inform DER -out /tmp/cernrootca2.crt \
+    && curl -fsSL 'https://cafiles.cern.ch/cafiles/certificates/CERN%20Grid%20Certification%20Authority(1).crt' -o /tmp/cerngridca.crt \
+    && curl -fsSL 'https://cafiles.cern.ch/cafiles/certificates/CERN%20Certification%20Authority.crt' -o /tmp/cernca.crt \
+    && cat /tmp/cernrootca2.crt >> /certs/rucio_ca.pem \
+    && cat /tmp/cerngridca.crt >> /certs/rucio_ca.pem \
+    && cat /tmp/cernca.crt >> /certs/rucio_ca.pem \
+    && rm /tmp/*.crt \
+    && update-ca-certificates
+
+ADD asyncssh_config.py /opt/ssh/jupyterhub-singleuser
+RUN fix-permissions /opt/ssh/jupyterhub-singleuser \
+    && chown -R $NB_UID /opt/ssh/jupyterhub-singleuser \
+    && chmod +x /opt/ssh/jupyterhub-singleuser
+
+# Setup extension Rucio instance config
+ADD configure.py /opt/setup-rucio-jupyterlab/configure.py
+RUN fix-permissions /opt/setup-rucio-jupyterlab/configure.py \
+    && chown -R $NB_UID /opt/setup-rucio-jupyterlab/configure.py \
+    && chmod +x /opt/setup-rucio-jupyterlab/configure.py
+
+ADD setup.sh /usr/local/bin/setup.sh
+RUN fix-permissions /usr/local/bin/setup.sh \
+    && sed -i -e 's/\r$/\n/' /usr/local/bin/setup.sh \
+    && chmod +x /usr/local/bin/setup.sh
+
+RUN mkdir -p /opt/rucio/etc \
+#    && touch /opt/rucio/etc/rucio.cfg \
+    && fix-permissions /opt/rucio/etc \
+    && chown -R ${NB_UID}:${NB_GID} /opt/rucio/etc 
+
+#    && /usr/local/bin/setup.sh 
+#RUN chown -R $NB_UID $HOME/.jupyter/jupyter_notebook_config.json 
+#    && chown -R $NB_UID /etc/jupyter/jupyter_notebook_config.py
+
+
+#ENV IPYTHONDIR=/etc/ipython
+#ADD ipython_kernel_config.json /etc/ipython/profile_default/ipython_kernel_config.json
+#RUN chown -R $NB_UID /etc/ipython
+ENV JUPYTER_ENABLE_LAB=yes
+
+WORKDIR $HOME
+USER $NB_UID
+
+CMD ["setup.sh", "start-notebook.sh"]

vre-singleuser-itwinai/asyncssh_config.py

@@ -0,0 +1,55 @@
+#!/opt/conda/bin/python
+# -*- coding: utf-8 -*-
+#
+# D. Ciangottini
+#
+import re
+import sys
+from jupyterhub.singleuser import main
+import asyncssh
+import logging
+import sys
+import asyncio
+import os
+from subprocess import Popen
+
+ssh_host = os.environ.get("JHUB_HOST")
+ssh_url_port = os.environ.get("SSH_PORT")
+username = os.environ.get("JUPYTERHUB_USER")
+token = os.environ.get("JUPYTERHUB_API_TOKEN")
+
+fwd_port = os.environ.get("FWD_PORT")
+
+async def run_client():
+    async with asyncssh.connect(
+                    host=ssh_host,
+                    port=int(ssh_url_port),
+                    username=username,
+                    password=token,
+                    known_hosts=None,
+                ) as conn:
+        conn.set_keepalive(interval=14.0, count_max=10)
+        listener = await conn.forward_remote_port(
+                "0.0.0.0",
+                int(fwd_port),
+                "0.0.0.0",
+                int(fwd_port),
+        )
+        await listener.wait_closed()
+
+
+if __name__ == '__main__':
+    print("Connecting ssh...")
+    loop = asyncio.get_event_loop()
+    loop.create_task(run_client())
+
+    print("Configuring Rucio extension...")
+    p = Popen(['/usr/local/bin/setup.sh'])
+    while p.poll() is None:
+        pass
+
+    print("Starting JLAB")
+    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+    sys.exit(main())
+
+

vre-singleuser-itwinai/configure.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python
+# Derived from https://gitlab.cern.ch/escape-wp2/docker-images/-/blob/master/datalake-singleuser/bin/configure.py (restricted access).
+
+import os
+import json
+import configparser
+
+#HOME = '/home/jovyan'
+#HOME = '/ceph/hpc/home/ciangottinid'
+
+def write_jupyterlab_config():
+    HOME = os.getenv('HOME', '/ceph/hpc/home/ciangottinid')
+
+    file_path = HOME + '/.jupyter/jupyter_notebook_config.json'
+    if not os.path.isfile(file_path):
+        os.makedirs(HOME + '/.jupyter/', exist_ok=True)
+    else:
+        config_file = open(file_path, 'r')
+        config_payload = config_file.read()
+        config_file.close()
+
+    try:
+        config_json = json.loads(config_payload)
+    except:
+        config_json = {}
+
+# Looking to the rucio-jupyterlab configuration; https://github.com/rucio/jupyterlab-extension/blob/master/rucio_jupyterlab/config/schema.py#L101
+#  either ("destination_rse", "rse_mount_path") either ("rucio_ca_cert") are required env vars, even if they are defined in the jhub manifest.
+# Adding 'rucio_base_url' too - from debugging experience
+
+    # instance_config = {
+    #     "name": os.getenv('RUCIO_NAME', 'default'),
+    #     "display_name": os.getenv('RUCIO_DISPLAY_NAME', 'Default Instance'),
+    #     "rucio_base_url": os.getenv('RUCIO_BASE_URL', 'DEFAULT rucio base url'),
+    #     "rucio_auth_url": os.getenv('RUCIO_AUTH_URL'),
+    #     "rucio_webui_url": os.getenv('RUCIO_WEBUI_URL'),
+    #     "rucio_ca_cert": os.getenv('RUCIO_CA_CERT', 'cacert.pem'),
+    #     "site_name": os.getenv('RUCIO_SITE_NAME'),
+    #     "vo": os.getenv('RUCIO_VO'),
+    #     "voms_enabled": os.getenv('RUCIO_VOMS_ENABLED', '0') == '1',
+    #     "voms_vomses_path": os.getenv('RUCIO_VOMS_VOMSES_PATH'),
+    #     "voms_certdir_path": os.getenv('RUCIO_VOMS_CERTDIR_PATH'),
+    #     "voms_vomsdir_path": os.getenv('RUCIO_VOMS_VOMSDIR_PATH'),
+    #     "destination_rse": os.getenv('RUCIO_DESTINATION_RSE', 'DEFAULT rse destination'),
+    #     "rse_mount_path": os.getenv('RUCIO_RSE_MOUNT_PATH', 'DEFAULT rse mount path'),
+    #     "replication_rule_lifetime_days": int(os.getenv('RUCIO_REPLICATION_RULE_LIFETIME_DAYS')) if os.getenv('RUCIO_REPLICATION_RULE_LIFETIME_DAYS') else None,
+    #     "path_begins_at": int(os.getenv('RUCIO_PATH_BEGINS_AT', '0')),
+    #     "mode": os.getenv('RUCIO_MODE', 'replica'),
+    #     "wildcard_enabled": os.getenv('RUCIO_WILDCARD_ENABLED', '0') == '1',
+    #     "oidc_auth": os.getenv('RUCIO_OIDC_AUTH'),
+    #     "oidc_env_name": os.getenv('RUCIO_OIDC_ENV_NAME'),
+    #     "oidc_file_name": os.getenv('RUCIO_OIDC_FILE_NAME'),
+    # }
+
+    # instance_config = {k: v for k,
+    #                    v in instance_config.items() if v is not None}
+    # config_json['RucioConfig'] = {
+    #     'instances': [instance_config],
+    #     "default_instance": os.getenv('RUCIO_DEFAULT_INSTANCE',),
+    #     "default_auth_type": os.getenv('RUCIO_DEFAULT_AUTH_TYPE'),
+    # }
+
+    # Debugging and hardcoding from here
+
+    instance_config = {
+        "name": "rucio-intertwin-testbed.desy.de",
+        "display_name": "interTwinRUCIO",
+        "rucio_base_url": "https://rucio-intertwin-testbed.desy.de",
+        "rucio_auth_url": "https://rucio-intertwin-testbed-auth.desy.de",
+        "rucio_ca_cert": "/opt/conda/lib/python3.9/site-packages/certifi/cacert.pem",
+        "site_name": "VEGA",
+        "voms_enabled": os.getenv('RUCIO_VOMS_ENABLED', '0') == '1',
+        "destination_rse": "VEGA-DCACHE",
+        "rse_mount_path": "/dcache/sling.si/projects/intertwin",
+        "path_begins_at": 4,
+        "mode": "replica", 
+        #"mode": "download", 
+        "wildcard_enabled": os.getenv('RUCIO_WILDCARD_ENABLED', '0') == '0',
+        "oidc_auth": "env",
+        "oidc_env_name": "RUCIO_ACCESS_TOKEN"
+      }
+
+    instance_config = {k: v for k,
+                       v in instance_config.items() if v is not None}
+    config_json['RucioConfig'] = {
+        'instances': [instance_config],
+        "default_instance": os.getenv('RUCIO_DEFAULT_INSTANCE', 'rucio-intertwin-testbed.desy.de'),
+        "default_auth_type": os.getenv('RUCIO_DEFAULT_AUTH_TYPE', 'oidc'),
+    }
+
+    # up to here
+
+    config_file = open(file_path, 'w')
+    config_file.write(json.dumps(config_json, indent=2))
+    config_file.close()
+
+def write_rucio_config():
+
+    rucio_config = configparser.ConfigParser()
+
+    client_config = {
+        'rucio_host': os.getenv('RUCIO_BASE_URL', 'https://rucio-intertwin-testbed.desy.de'),
+        'auth_host': os.getenv('RUCIO_AUTH_URL', 'https://rucio-intertwin-testbed-auth.desy.de'),
+        'ca_cert': os.getenv('RUCIO_CA_CERT', '/certs/rucio_ca.pem'),
+        'auth_type': os.getenv('RUCIO_AUTH_TYPE', 'oidc'), # 'x509' or 'oidc'
+        'account': os.getenv('RUCIO_ACCOUNT', '$RUCIO_ACCOUNT'), # This is the RUCIO account name, need to be mapped from idp
+        'oidc_polling': 'true',
+        'oidc_scope': 'openid profile offline_access eduperson_entitlement',
+        #'username': os.getenv('RUCIO_USERNAME', ''),
+        #'password': os.getenv('RUCIO_PASSWORD', ''),
+        'auth_token_file_path': '/tmp/rucio_oauth.token',
+        'request_retries': 3,
+        'protocol_stat_retries': 6
+    }
+    client_config = dict((k, v) for k, v in client_config.items() if v)
+    rucio_config['client'] = client_config
+
+    if not os.path.isfile('/opt/rucio/etc/rucio.cfg'):
+        os.makedirs('/opt/rucio/etc/', exist_ok=True)
+
+    with open('/opt/rucio/etc/rucio.cfg', 'w') as f:
+        rucio_config.write(f)    
+
+if __name__ == '__main__':
+    write_jupyterlab_config()
+    #write_rucio_config()

vre-singleuser-itwinai/setup.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+python /opt/setup-rucio-jupyterlab/configure.py
+
+# Creation of the rucio.cfg file
+mkdir -p /certs /tmp;
+echo -n $RUCIO_ACCESS_TOKEN > /tmp/rucio_oauth.token;
+# mkdir -p /opt/rucio/etc;
+# echo "[client]" >> /opt/rucio/etc/rucio.cfg;
+# echo "rucio_host = https://rucio-intertwin-testbed.desy.de" >> /opt/rucio/etc/rucio.cfg;
+# echo "auth_host = https://rucio-intertwin-testbed-auth.desy.de" >> /opt/rucio/etc/rucio.cfg;
+# #echo "ca_cert = /certs/rucio_ca.pem" >> /opt/rucio/etc/rucio.cfg;
+# echo "ca_cert = /opt/conda/lib/python3.9/site-packages/certifi/cacert.pem" >> /opt/rucio/etc/rucio.cfg;
+# echo "account = $JUPYTERHUB_USER" >> /opt/rucio/etc/rucio.cfg;
+# echo "auth_type = oidc" >> /opt/rucio/etc/rucio.cfg;
+# echo "oidc_audience = rucio-testbed" >> /opt/rucio/etc/rucio.cfg;
+# echo "oidc_polling = true" >> /opt/rucio/etc/rucio.cfg;
+# echo "oidc_scope = openid profile offline_access eduperson_entitlement" >> /opt/rucio/etc/rucio.cfg;
+# echo "auth_token_file_path = /tmp/rucio_oauth.token" >> /opt/rucio/etc/rucio.cfg;
+
+exec "$@"