release: v1.0.0

.github/workflows/tests.yml

@@ -19,11 +19,13 @@ name: CI
 
 on:
   push:
-    branches: main
+    branches:
+      - main
+      - staging
   pull_request:
     branches:
       - main
-      - "maint-**"
+      - "staging"
   schedule:
     # * is a special character in YAML so you have to quote this string
     - cron: "0 3 * * 6"
@@ -36,29 +38,21 @@ on:
 
 jobs:
   Tests:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         # TODO: You can add/remove combinations e.g. `dev` requirements or
         # `postgresql13` by adding a new item to the following lists.
         # You can see the complete list of services and versions that are available at:
         # https://docker-services-cli.readthedocs.io/en/latest/configuration.html
         python-version: ["3.9"]
-        requirements-level: [pypi]
         db-service: [postgresql13]
         search-service: [elasticsearch]
-        include:
-          - search-service: elasticsearch
-            SEARCH_EXTRAS: "elasticsearch"
-
-          # - search-service: opensearch2
-          #   SEARCH_EXTRAS: "opensearch2"
+        dependencies: ['dev', 'deploy']
 
     env:
       DB: ${{ matrix.db-service }}
       SEARCH: ${{ matrix.search-service }}
-      # TODO: Adapt EXTRAS accordingly
-      EXTRAS: tests,${{ matrix.search-service }}
 
     steps:
       - name: Checkout
@@ -69,25 +63,20 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Generate dependencies
-        run: |
-          python -m pip install --upgrade pip setuptools py wheel requirements-builder
-          requirements-builder -e "$EXTRAS" --level=${{ matrix.requirements-level }} setup.py > .${{ matrix.requirements-level }}-${{ matrix.python-version }}-requirements.txt
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
 
-      - name: Cache pip
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('.${{ matrix.requirements-level }}-${{ matrix.python-version }}-requirements.txt') }}
+      - name: Update depencencies
+        if: ${{ matrix.dependencies == 'dev' }}
+        run: poetry update
 
       - name: Install dependencies
         run: |
-          pip install -r .${{matrix.requirements-level}}-${{ matrix.python-version }}-requirements.txt
-          pip install ".[$EXTRAS]"
-          pip freeze
-          docker --version
-          docker-compose --version
+          poetry install
 
       - name: Run tests
         run: |
-          ./run-tests.sh
+          poetry run ./run-tests.sh
+
+      - name: Run build
+        run: poetry build

README.rst

@@ -31,8 +31,3 @@
         :target: https://github.com/rero/rero-invenio-files/blob/master/LICENSE
 
 Files support for the RERO invenio instances.
-
-TODO: Please provide feature overview of module
-
-Further documentation is available on
-https://rero-invenio-files.readthedocs.io/

pyproject.toml

@@ -1,20 +1,20 @@
 [tool.poetry]
 name = "rero-invenio-files"
-version = "0.1.0"
+version = "1.0.0"
 description = "Files support for the RERO invenio instances."
 authors = ["RERO <software@rero.ch>"]
 license = "GNU Affero General Public License v3.0"
 readme = "README.rst"
 
 [tool.poetry.dependencies]
 python = "^3.9"
-invenio-records-resources = "^4.18.3"
+invenio-records-resources = "<5.7.0"
 invenio-search = {version = ">=2.1.0,<3.0.0", extras = ["elasticsearch7"]}
 invenio-db = {version = ">=1.1.0,<1.2.0", extras = ["postgresql"]}
 fpdf2 = "^2.7.7"
 pymupdf = "^1.23.21"
 invenio-previewer = "^2.2.0"
-invenio-theme = ">=2.5.7,<3.0.0"
+invenio-theme = "<4.0.0"
 
 
 [tool.poetry.group.dev.dependencies]
@@ -23,6 +23,8 @@ pytest-invenio = ">=2.1.6,<3.0.0"
 pytest-black = ">=0.3.0"
 sphinx = ">=4.5.0"
 mock = "^5.1.0"
+safety = "^3.2.0"
+autoflake = "^2.3.1"
 
 [build-system]
 requires = ["poetry-core"]
@@ -50,3 +52,16 @@ records = "rero_invenio_files.records.jsonschemas"
 
 [tool.poetry.plugins."invenio_search.mappings"]
 records = "rero_invenio_files.records.mappings"
+
+[tool.distutils.bdist_wheel]
+universal = true
+
+[tool.pydocstyle]
+add_ignore = "D401"
+
+[tool.isort]
+profile="black"
+
+[tool.pytest.ini_options]
+addopts = "--black --isort --pydocstyle --doctest-glob=\"*.rst\" --doctest-modules --cov=rero_invenio_files --cov-report=term-missing"
+testpaths = "docs tests rero_invenio_files"

rero_invenio_files/records/components.py

@@ -17,6 +17,7 @@
 
 """Thumbnail generation and full text extraction component."""
 
+import contextlib
 import os
 from io import BytesIO
 
@@ -115,55 +116,57 @@ def commit_file(self, identity, id_, file_key, record):
         sf = self.service
         recid = record.pid.pid_value
         # thumbnail
-        try:
-            blob = self.create_thumbnail_from_file(rfile.uri, rfile.mimetype)
-        except Exception:
-            blob = None
-        if blob:
-            thumb_name = self.change_filename_extension(file_key, "jpg")
-            sf.init_files(
-                identity=identity,
-                id_=recid,
-                data=[
-                    {"key": thumb_name, "type": "thumbnail", "thumbnail_for": file_key}
-                ],
-                uow=self.uow,
-            )
-            sf.set_file_content(
-                identity,
-                id_=recid,
-                file_key=thumb_name,
-                stream=BytesIO(blob),
-                uow=self.uow,
-            )
-            sf.commit_file(
-                identity=identity, id_=recid, file_key=thumb_name, uow=self.uow
-            )
+        with contextlib.suppress(Exception):
+            if blob := self.create_thumbnail_from_file(rfile.uri, rfile.mimetype):
+                thumb_name = self.change_filename_extension(file_key, "jpg")
+                sf.init_files(
+                    identity=identity,
+                    id_=recid,
+                    data=[
+                        {
+                            "key": thumb_name,
+                            "type": "thumbnail",
+                            "thumbnail_for": file_key,
+                        }
+                    ],
+                    uow=self.uow,
+                )
+                sf.set_file_content(
+                    identity,
+                    id_=recid,
+                    file_key=thumb_name,
+                    stream=BytesIO(blob),
+                    uow=self.uow,
+                )
+                sf.commit_file(
+                    identity=identity, id_=recid, file_key=thumb_name, uow=self.uow
+                )
         # fulltext
-        try:
-            fulltext = self.create_fulltext_from_file(rfile.uri, rfile.mimetype)
-        except Exception:
-            fulltext = None
-        if fulltext:
-            thumb_name = self.change_filename_extension(file_key, "txt")
-            sf.init_files(
-                identity=identity,
-                id_=recid,
-                data=[
-                    {"key": thumb_name, "type": "fulltext", "fulltext_for": file_key}
-                ],
-                uow=self.uow,
-            )
-            sf.set_file_content(
-                identity=identity,
-                id_=recid,
-                file_key=thumb_name,
-                stream=BytesIO(fulltext.encode()),
-                uow=self.uow,
-            )
-            sf.commit_file(
-                identity=identity, id_=recid, file_key=thumb_name, uow=self.uow
-            )
+        with contextlib.suppress(Exception):
+            if fulltext := self.create_fulltext_from_file(rfile.uri, rfile.mimetype):
+                thumb_name = self.change_filename_extension(file_key, "txt")
+                sf.init_files(
+                    identity=identity,
+                    id_=recid,
+                    data=[
+                        {
+                            "key": thumb_name,
+                            "type": "fulltext",
+                            "fulltext_for": file_key,
+                        }
+                    ],
+                    uow=self.uow,
+                )
+                sf.set_file_content(
+                    identity=identity,
+                    id_=recid,
+                    file_key=thumb_name,
+                    stream=BytesIO(fulltext.encode()),
+                    uow=self.uow,
+                )
+                sf.commit_file(
+                    identity=identity, id_=recid, file_key=thumb_name, uow=self.uow
+                )
 
     def delete_file(self, identity, id_, file_key, record, deleted_file):
         """Delete file handler.
@@ -180,16 +183,12 @@ def delete_file(self, identity, id_, file_key, record, deleted_file):
         sf = self.service
         recid = record.pid.pid_value
         thumb_name = self.change_filename_extension(file_key, "jpg")
-        try:
+        with contextlib.suppress(FileKeyNotFoundError):
             sf.delete_file(
                 identity=identity, id_=recid, file_key=thumb_name, uow=self.uow
             )
-        except FileKeyNotFoundError:
-            pass
         fulltext_name = self.change_filename_extension(file_key, "txt")
-        try:
+        with contextlib.suppress(FileKeyNotFoundError):
             sf.delete_file(
                 identity=identity, id_=recid, file_key=fulltext_name, uow=self.uow
             )
-        except FileKeyNotFoundError:
-            pass

rero_invenio_files/records/previewer.py

@@ -82,13 +82,7 @@ def preview(pid, record, template=None, **kwargs):
                 return plugin.preview(fileobj)
             except Exception:
                 current_app.logger.warning(
-                    (
-                        "Preview failed for {key}, in {pid_type}:{pid_value}".format(
-                            key=fileobj.file.key,
-                            pid_type=fileobj.pid.pid_type,
-                            pid_value=fileobj.pid.pid_value,
-                        )
-                    ),
+                    f"Preview failed for {fileobj.file.key}, in {fileobj.pid.pid_type}:{fileobj.pid.pid_value}",
                     exc_info=True,
                 )
     return default.preview(fileobj)

rero_invenio_files/records/schema.py

@@ -18,7 +18,7 @@
 """Files support for the RERO invenio instances."""
 
 from invenio_records_resources.services.records.schema import BaseRecordSchema
-from marshmallow import INCLUDE, Schema, fields
+from marshmallow import Schema, fields
 from marshmallow_utils.fields import SanitizedUnicode
 
 

rero_invenio_files/records/services.py

@@ -45,9 +45,7 @@ def should_render(self, obj, ctx):
         # pdf is supported
         if not hasattr(obj.file, "mimetype"):
             return False
-        if obj.file.mimetype in ["application/pdf", "image/jpeg", "image/png"]:
-            return True
-        return False
+        return obj.file.mimetype in ["application/pdf", "image/jpeg", "image/png"]
 
 
 class ThumbFileLink(PreviewFileLink):

run-tests.sh

@@ -35,6 +35,15 @@ trap cleanup EXIT
 # python -m check_manifest
 python -m sphinx.cmd.build -qnNW docs docs/_build/html
 
+safety_exceptions="-i 51668 -i 42194 -i 62019 -i 67599 -i 51457"
+msg=$(safety check -o text ${safety_exceptions}) || {
+    echo "Safety vulnerabilites found for packages:" $(safety check -o bare ${safety_exceptions})
+    echo "Run: \"safety check -o screen ${safety_exceptions} | grep -i vulnerability\" for more details"
+    exit 1
+  }
+
+autoflake -r --remove-all-unused-imports --ignore-init-module-imports --quiet .
+
 # TODO: Remove services below that are not neeed (fix also the usage note).
 eval "$(docker-services-cli up --db ${DB:-postgresql} --search ${SEARCH:-elasticsearch} --cache ${CACHE:-redis} --mq ${MQ:-rabbitmq} --env)"
 python -m pytest