Merge pull request #36 from bento-platform/misc-updates

Misc updates + extraneous debugpy warning fix

.github/workflows/build.yml

@@ -23,7 +23,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Run Bento build action
-        uses: bento-platform/bento_build_action@v0.6
+        uses: bento-platform/bento_build_action@v0.9.3
         with:
           registry: ghcr.io
           registry-username: ${{ github.actor }}

.github/workflows/lint.yml

@@ -8,11 +8,11 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         name: Set up Python
         with:
-          python-version: 3.6
+          python-version: "3.8"
       - name: Install flake8
         run: python -m pip install flake8
       - name: Run linter

.github/workflows/test.yml

@@ -9,10 +9,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ 3.6, 3.9 ]
+        python-version: [ "3.8", "3.10" ]
     steps:
-      - uses: actions/checkout@v1
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         name: Set up Python
         with:
           python-version: ${{ matrix.python-version }}
@@ -26,10 +26,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ 3.6, 3.9 ]
+        python-version: [ "3.8", "3.10" ]
     steps:
-      - uses: actions/checkout@v1
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         name: Set up Python
         with:
           python-version: ${{ matrix.python-version }}

.gitignore

@@ -11,10 +11,7 @@ chord_drs/startup.sh
 htmlcov/
 
 # IDE stuff
-.idea/$CACHE_FILE$
-.idea/dataSources*
-.idea/rSettings.xml
-.idea/workspace.xml
+.idea/*
 *.swp
 *.swo
 .vscode

Dockerfile

@@ -1,20 +1,22 @@
-FROM ghcr.io/bento-platform/bento_base_image:python-debian-2022.10.11
+FROM ghcr.io/bento-platform/bento_base_image:python-debian-2022.12.06
 
 # TODO: change USER
 USER root
 
 RUN apt install gcc libffi-dev -y
 
 RUN echo "Building DRS in Production Mode";
-WORKDIR /drs/bento_drs
+WORKDIR /drs
 RUN mkdir /wes && \
-    mkdir -p /drs/bento_drs/data/obj && \
-    mkdir -p /drs/bento_drs/data/db;
+    mkdir -p /drs/data/obj && \
+    mkdir -p /drs/data/db
+
+# Install dependencies
 COPY requirements.txt requirements.txt
 RUN ["pip", "install", "-r", "requirements.txt"]
-COPY . .
 
-# Run
-WORKDIR /drs/bento_drs/chord_drs
-COPY startup.sh ./startup.sh
-CMD ["sh", "startup.sh"]
+# Copy only what's required for a production instance
+COPY chord_drs chord_drs
+COPY entrypoint.bash .
+
+ENTRYPOINT ["/bin/bash", "./entrypoint.bash"]

chord_drs/app.py

@@ -51,13 +51,14 @@
 
 # # debugger section
 # Ensure 'debugpy' is installed (via requirements.txt or manually)
-DEBUG = os.environ.get('FLASK_DEBUG', False)
+DEBUG = os.environ.get("FLASK_DEBUG", "false").strip().lower() in ("true", "1")
 if DEBUG:
     try:
+        # noinspection PyPackageRequirements
         import debugpy
-        DEBUGGER_PORT = int(os.environ.get('DEBUGGER_PORT', 5678))
-        debugpy.listen(("0.0.0.0", DEBUGGER_PORT))
-        print('\nDebugger Attached\n')
+        debugger_port = int(os.environ.get("DEBUGGER_PORT", "5678"))
+        debugpy.listen(("0.0.0.0", debugger_port))
+        application.logger.info("Debugger attached")
     except ImportError:
-        print("\nWARNING Module debugpy not found. To enable VSCode debugging, run:\n\tpip install debugpy\n")
+        application.logger.warning("Module debugpy not found. To enable VSCode debugging, run `pip install debugpy`")
 # # end debugger section

chord_drs/config.py

@@ -51,7 +51,7 @@ class Config:
     SQLALCHEMY_TRACK_MODIFICATIONS = False
     CHORD_URL: Optional[str] = os.environ.get("CHORD_URL")
     CHORD_SERVICE_URL_BASE_PATH: Optional[str] = os.environ.get("SERVICE_URL_BASE_PATH")
-    SERVICE_ID: str = os.environ.get("SERVICE_ID", SERVICE_TYPE)
+    SERVICE_ID: str = os.environ.get("SERVICE_ID", ":".join(list(SERVICE_TYPE.values())[:2]))
     SERVICE_DATA_SOURCE: str = DATA_SOURCE_MINIO if MINIO_URL else DATA_SOURCE_LOCAL
     SERVICE_DATA: Optional[str] = None if MINIO_URL else SERVICE_DATA
     MINIO_URL: Optional[str] = MINIO_URL

chord_drs/constants.py

@@ -1,7 +1,17 @@
 from chord_drs import __version__
 
-__all__ = ["SERVICE_NAME", "SERVICE_ARTIFACT", "SERVICE_TYPE"]
+__all__ = [
+    "BENTO_SERVICE_KIND",
+    "SERVICE_NAME",
+    "SERVICE_ARTIFACT",
+    "SERVICE_TYPE",
+]
 
+BENTO_SERVICE_KIND = "drs"
 SERVICE_NAME = "Bento Data Repository Service"
-SERVICE_ARTIFACT = "drs"
-SERVICE_TYPE = f"ca.c3g.chord:{SERVICE_ARTIFACT}:{__version__}"
+SERVICE_ARTIFACT = BENTO_SERVICE_KIND
+SERVICE_TYPE = {
+    "group": "ca.c3g.chord",
+    "artifact": SERVICE_ARTIFACT,
+    "version": __version__,
+}

chord_drs/package.cfg

@@ -1,5 +1,5 @@
 [package]
 name = chord_drs
-version = 0.7.0
+version = 0.8.0
 authors = Simon Chénard, David Lougheed
 author_emails = simon.chenard2@mcgill.ca, david.lougheed@mail.mcgill.ca

chord_drs/routes.py

@@ -17,7 +17,7 @@
 from urllib.parse import urljoin, urlparse
 
 from chord_drs import __version__
-from chord_drs.constants import SERVICE_NAME, SERVICE_TYPE
+from chord_drs.constants import BENTO_SERVICE_KIND, SERVICE_NAME, SERVICE_TYPE
 from chord_drs.data_sources import DATA_SOURCE_LOCAL, DATA_SOURCE_MINIO
 from chord_drs.db import db
 from chord_drs.models import DrsObject, DrsBundle
@@ -153,24 +153,29 @@ def service_info():
         "description": "Data repository service (based on GA4GH's specs) for a Bento platform node.",
         "organization": {
             "name": "C3G",
-            "url": "http://c3g.ca"
+            "url": "https://www.computationalgenomics.ca"
         },
-        "contactUrl": "mailto:simon.chenard2@mcgill.ca",
+        "contactUrl": "mailto:info@c3g.ca",
         "version": __version__,
-        "environment": "prod"
-        }
+        "environment": "prod",
+        "bento": {
+            "serviceKind": BENTO_SERVICE_KIND,
+        },
+    }
 
     if not current_app.config["BENTO_DEBUG"]:
         return jsonify(info)
 
     info["environment"] = "dev"
     try:
-        res_tag = subprocess.check_output(["git", "describe", "--tags", "--abbrev=0"])
-        if res_tag:
-            info["git_tag"] = res_tag.decode().rstrip()
-        res_branch = subprocess.check_output(["git", "branch", "--show-current"])
-        if res_branch:
-            info["git_branch"] = res_branch.decode().rstrip()
+        if res_tag := subprocess.check_output(["git", "describe", "--tags", "--abbrev=0"]):
+            res_tag_str = res_tag.decode().rstrip()
+            info["git_tag"] = res_tag_str
+            info["bento"]["gitTag"] = res_tag_str
+        if res_branch := subprocess.check_output(["git", "branch", "--show-current"]):
+            res_branch_str = res_branch.decode().strip()
+            info["git_branch"] = res_branch_str
+            info["bento"]["gitBranch"] = res_branch_str
 
     except Exception as e:
         except_name = type(e).__name__
@@ -248,7 +253,7 @@ def object_download(object_id):
             return send_file(
                 drs_object.location,
                 mimetype=MIME_OCTET_STREAM,
-                attachment_filename=drs_object.name,
+                download_name=drs_object.name,
             )
 
         current_app.logger.debug(f"Found Range header: {range_header}")
@@ -304,7 +309,7 @@ def generate_bytes():
             minio_obj["Body"],
             mimetype="application/octet-stream",
             as_attachment=True,
-            attachment_filename=drs_object.name
+            download_name=drs_object.name
         )
     )
 

dev.Dockerfile

@@ -1,18 +1,20 @@
-FROM ghcr.io/bento-platform/bento_base_image:python-debian-2022.10.11
+FROM ghcr.io/bento-platform/bento_base_image:python-debian-2022.12.06
 
 # TODO: change USER
 USER root
 
 RUN apt install libffi-dev -y
 
 RUN echo "Building DRS in Development Mode";
-WORKDIR /drs/bento_drs
+WORKDIR /drs
 RUN mkdir /wes && \
-    mkdir -p /drs/bento_drs/data/obj && \
-    mkdir -p /drs/bento_drs/data/db;
-COPY ./requirements.txt .
-RUN ["pip", "install", "debugpy", "-r", "requirements.txt"]
+    mkdir -p /drs/data/obj && \
+    mkdir -p /drs/data/db;
 
-# Run
-WORKDIR /drs/bento_drs/chord_drs
-COPY startup.sh ./startup.sh
+# Install requirements
+COPY requirements.txt .
+RUN pip install debugpy -r requirements.txt
+
+# Don't copy anything in - the dev compose file will mount the repo
+
+ENTRYPOINT ["/bin/bash", "./entrypoint.dev.bash"]

entrypoint.bash

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# CWD: /drs
+
+mkdir -p /drs/data/db
+mkdir -p /drs/data/obj
+
+export FLASK_APP="chord_drs.app:application"
+if [ -z "${INTERNAL_PORT}" ]; then
+  # Set default internal port to 5000
+  export INTERNAL_PORT=5000
+fi
+
+flask db upgrade
+
+# using 1 worker, multiple threads
+# see https://stackoverflow.com/questions/38425620/gunicorn-workers-and-threads
+gunicorn "${FLASK_APP}" -w 1 --threads $(expr 2 \* $(nproc --all) + 1) -b "0.0.0.0:${INTERNAL_PORT}"

entrypoint.dev.bash

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+export FLASK_ENV=development
+export FLASK_APP=chord_drs.app:application
+
+if [ -z "${INTERNAL_PORT}" ]; then
+  # Set default internal port to 5000
+  export INTERNAL_PORT=5000
+fi
+
+python -m pip install -r requirements.txt
+
+flask db upgrade
+
+python -m debugpy --listen 0.0.0.0:5678 -m flask run --host 0.0.0.0 --port "${INTERNAL_PORT}"

requirements.txt

@@ -1,25 +1,32 @@
+aiohttp==3.8.3
+aiosignal==1.3.1
 alembic==1.7.1
 appdirs==1.4.4
+async-timeout==4.0.2
 attrs==21.2.0
 backports.entry-points-selectable==1.1.0
-bento-lib==3.0.1
+bento-lib==5.2.0
 boto3==1.18.34
 botocore==1.21.34
-certifi==2021.5.30
+cachetools==5.2.1
+certifi==2022.12.7
 cffi==1.14.6
-chardet==4.0.0
+chardet==5.1.0
 charset-normalizer==2.0.4
-click==8.0.1
+click==8.1.3
 codecov==2.1.12
+colorama==0.4.6
 coverage==5.5
 cryptography==3.4.8
-distlib==0.3.2
-filelock==3.0.12
-flake8==3.9.2
+distlib==0.3.6
+exceptiongroup==1.1.0
+filelock==3.9.0
+flake8==6.0.0
 flake8-polyfill==1.0.2
-Flask==2.0.1
+Flask==2.2.2
 Flask-Migrate==3.1.0
 Flask-SQLAlchemy==2.5.1
+frozenlist==1.3.3
 greenlet==2.0.1
 idna==2.10
 iniconfig==1.1.1
@@ -28,43 +35,47 @@ Jinja2==3.0.1
 jmespath==0.10.0
 jsonschema==3.2.0
 Mako==1.1.5
-MarkupSafe==2.0.1
-mccabe==0.6.1
+MarkupSafe==2.1.1
+mccabe==0.7.0
 mock==4.0.3
-more-itertools==8.8.0
+more-itertools==9.0.0
 moto==2.2.6
-packaging==21.0
+multidict==6.0.4
+packaging==23.0
 pep8-naming==0.12.1
-platformdirs==2.3.0
-pluggy==0.13.1
+platformdirs==2.6.2
+pluggy==1.0.0
 prometheus-client==0.9.0
-prometheus-flask-exporter==0.14.1
-psycopg2-binary==2.9.1
+prometheus-flask-exporter==0.21.0
+psycopg2-binary==2.9.5
 py==1.10.0
-pycodestyle==2.7.0
+pycodestyle==2.10.0
 pycparser==2.20
-pyflakes==2.3.1
+pyflakes==3.0.1
 pyparsing==2.4.7
+pyproject_api==1.4.0
 pyrsistent==0.18.0
-pytest==6.2.5
-pytest-cov==2.12.1
+pytest==7.2.0
+pytest-cov==4.0.0
 pytest-dotenv==0.5.2
 pytest-lazy-fixture==0.6.3
 python-dateutil==2.8.2
-python-dotenv==0.19.0
+python-dotenv==0.21.0
 python-editor==1.0.4
 pytz==2021.1
 PyYAML==5.4.1
 redis==3.5.3
-requests==2.26.0
+requests==2.28.2
 responses==0.13.4
 s3transfer==0.5.0
 six==1.16.0
-SQLAlchemy==1.4.23
+SQLAlchemy==1.4.46
 toml==0.10.2
-tox==3.24.3
-urllib3==1.26.6
-virtualenv==20.7.2
-Werkzeug==2.0.1
+tomli==2.0.1
+tox==4.2.8
+urllib3==1.26.14
+virtualenv==20.17.1
+Werkzeug==2.2.2
 xmltodict==0.12.0
+yarl==1.8.2
 zipp==3.5.0