From 9668442aa294003164a759b949f968b5024961d4 Mon Sep 17 00:00:00 2001 From: Zacharias Zacharodimos Date: Fri, 2 Aug 2024 15:54:34 +0200 Subject: [PATCH] cleanup leftovers and typo --- .github/workflows/tests.yml | 5 - INSTALL.rst | 2 +- Upgrade.txt | 221 ------------------------------------ cds/config.py | 1 - cds/modules/deposit/api.py | 1 - setup.cfg | 2 +- 6 files changed, 2 insertions(+), 230 deletions(-) delete mode 100644 Upgrade.txt diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1d50a1616..c6a39d296 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -74,11 +74,6 @@ jobs: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('.pypi-${{ matrix.python-version }}-requirements.txt') }} - # - name: Use Node.js ${{ matrix.node-version }} - # uses: actions/setup-node@v3 - # with: - # node-version: ${{ matrix.node-version }} - - name: Install dependencies run: | pip install -r .pypi-${{ matrix.python-version }}-requirements.txt diff --git a/INSTALL.rst b/INSTALL.rst index c7fdae96b..88db02ddc 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -29,7 +29,7 @@ If you get an empty output from the previous command, run: .. code-block:: console - (cds3)$ mkdir src ; cd srcbt + (cds3)$ mkdir src ; cd src (cds3)$ git clone https://github.com/CERNDocumentServer/cds-videos.git 5. Run a script to set up some npm configurations and install some packages: diff --git a/Upgrade.txt b/Upgrade.txt deleted file mode 100644 index 60be3e3d5..000000000 --- a/Upgrade.txt +++ /dev/null @@ -1,221 +0,0 @@ -# Notes - -- [x] type: completion for suggestions -- [x] ffmpeg: gif method was moved in module -- [x] admin panel shows Banners -- [x] test statistics, should we install raven? -- [x] missing os-v2 invenio_opendefinition.mappings -- [x] existing kw need updating, remove payload as it is not supported -- project deletion from the UI fails to index the project as it tries to index it by id which results to a PIDDeletedError -- cds-js module to be archived -- [x] integrate invenio-stats and adapt code accordingly -- [x] check https://cern.service-now.com/service-portal?id=outage&n=OTG0150438 - - check krb ticket by running 'klist -e -c /var/run/krb5-tokens/token.krb5' - -# DB update - -- if local users then we need to re-encrypt the passwords - - NOT needed if set SECURITY_PASSWORD_SINGLE_HASH = True -- invenio alembic upgrade -- Remove keywords.suggest_name.payload -``` -from sqlalchemy.orm.exc import NoResultFound - -from cds.modules.records.api import Keyword -from invenio_pidstore.models import PersistentIdentifier, PIDStatus -from invenio_db import db - - -def get_all_pids_by(pid_type): - """Get all PIDs for the given type. - - :param pid_type: String representing the PID type value, for example 'recid'. - """ - pids = PersistentIdentifier.query.filter(PersistentIdentifier.pid_type == pid_type).filter( - PersistentIdentifier.status == PIDStatus.REGISTERED).yield_per(100) - return pids - - -def get_record_by_pid(pid): - """Fetch and return the published record given its PID. - - :param pid: String representing the PID value. - """ - try: - return Keyword.get_record(pid.object_uuid) - except NoResultFound: - return None - - -kwds = get_all_pids_by("kwid") - -for pid in kwds: - rec = get_record_by_pid(pid) - if rec is not None: - rec["suggest_name"].pop("payload", None) - rec.commit() -db.session.commit() -``` - -# Search reindex - -```bash - -cds index reindex -t recid -cds index reindex -t depid -cds index reindex -t catid -cds index reindex -t kwid -cds index reindex -t od_lic -cds index run - -``` - -## Local setup - -- Node v18 -- Python 3.9 - -```bash - -# setup -./scripts/bootstrap_upgrade -./scripts/setup-instance.sh - -# web server -./scripts/server - -# celery -./scripts/celery - -# assets watch -./scripts/assets-watch - -``` - -## REST API upload - -### Upload a remote file - -```invenio shell - -import requests - -# Create a personal access token in /account/settings/applications/tokens/new -# ticking all scopes -# AUTH_TOKEN = "4s9XMJcFsGiKH9GSwU004kHb68W7tlyLnsweZ970TBsceQCALwWLwRxR4ujf" -# SERVER_HOST = "https://127.0.0.1:5000" -AUTH_TOKEN = "62Ak9JxKah967a3tuuFd0izyyijutFdvUvzjNEFKlc3smcJGWcNf0lebv89s" -SERVER_HOST = "https://127.0.0.1:5000" -video_url = "https://cernbox.cern.ch/remote.php/dav/public-files/tt6kuZcrloXWhVs/20170205%20-%20FOSDEM%202017%20-%20Magnum%20at%20CERN.mp4" - -def upload_video(file_path, auth_token): - headers = {"Authorization": f"Bearer {auth_token}"} - headers["content-type"] = "application/vnd.project.partial+json" - body = { - "$schema": f"{SERVER_HOST}/schemas/deposits/records/videos/project/project-v1.0.0.json", - } - res_project = requests.post( - f"{SERVER_HOST}/api/deposits/project/", - json=body, - headers=headers, - verify=False if "127.0.0.1" in SERVER_HOST else True, - ) - - res_project = requests.put( - f"{SERVER_HOST}/api/deposits/project/{res_project.json()['id']}", - json={ - "_access": {"update": ["Tim.Bell@cern.ch"]}, - "category": "CERN", - "type": "VIDEO" - }, - headers=headers, - verify=False if "127.0.0.1" in SERVER_HOST else True, - ) - - # Video - headers["content-type"] = "application/vnd.video.partial+json" - body = { - "$schema": f"{SERVER_HOST}/schemas/deposits/records/videos/video/video-v1.0.0.json", - } - body["_project_id"] = res_project.json()["id"] - - res_video = requests.post( - f"{SERVER_HOST}/api/deposits/video/", - json=body, - headers=headers, - verify=False if "127.0.0.1" in SERVER_HOST else True, - ) - - res_video = requests.put( - f"{SERVER_HOST}/api/deposits/video/{res_video.json()['id']}", - json={ - "_access": {"update": ["Tim.Bell@cern.ch"]}, - "title": {"title": file_path.split("/")[-1].split(".")[0]}, - "contributors": [ - { - "email": "gabrielle.de.bonnerive@cern.ch", - "ids": [{"source": "cern", "value": "832317"}], - "name": "Pepin Fontaine De Bonnerive, Gabrielle", - "role": "Editor", - }, - { - "affiliations": ["CERN"], - "email": "Paola.Catapano@cern.ch", - "ids": [ - {"source": "cern", "value": "380837"}, - {"source": "cds", "value": "2050975"}, - ], - "name": "Catapano, Paola", - "role": "Screenwriter", - }, - ], - "copyright": { - "holder": "CERN", - "url": "http://copyright.web.cern.ch", - "year": "2024", - }, - "date": "2024-05-17", - "description": "Visual curriculum vitae of Fabiola Gianotti at CERN, from her participation in the UA2 experiment to the election for her second mandate as CERN Director General.", - "doi": "10.17181/videos.2300133", - "duration": "00:01:49", - "featured": False, - "keywords": [ - {"name": "Fabiola"}, - {"name": "Gianotti"}, - {"name": "Director"}, - {"name": "General"}, - {"name": "biography"}, - {"name": "slideshow"}, - ], - "language": "en", - "license": [{"license": "CERN", "url": "http://copyright.web.cern.ch"}], - "publication_date": "2024-05-22", - "title": {"title": "Fabiola Gianotti Short Visual CV 2024"}, - "type": "VIDEO", - "vr": False, - }, - headers=headers, - verify=False if "127.0.0.1" in SERVER_HOST else True, - ) - # File upload - headers["content-type"] = "application/json" - body = { - "uri": file_path, - "key": file_path.split("/")[-1], - "bucket_id": res_video.json()["metadata"]["_buckets"]["deposit"], - "deposit_id": res_video.json()["metadata"]["_deposit"]["id"], - } - res_flow = requests.post( - f"{SERVER_HOST}/api/flows/", - json=body, - headers=headers, - verify=False if "127.0.0.1" in SERVER_HOST else True, - ).json() - flow_id = res_flow["tags"]["flow_id"] - - video_link = res_project.json()["links"]["html"] - print(f"Uploaded video: {file_path}. Link: {video_link}") - return video_link, flow_id - -upload_video(video_url, AUTH_TOKEN) -``` \ No newline at end of file diff --git a/cds/config.py b/cds/config.py index 42c3e2fd3..9287ca858 100644 --- a/cds/config.py +++ b/cds/config.py @@ -1105,7 +1105,6 @@ def _parse_env_bool(var_name, default=None): "base-uri": ["'self'"], "worker-src": ["'self'", "blob:"], "manifest-src": ["'none'"], - # "prefetch-src": ["'none'"], "font-src": [ "'self'", "data:", diff --git a/cds/modules/deposit/api.py b/cds/modules/deposit/api.py index 1a58bdfb4..00f94ef22 100644 --- a/cds/modules/deposit/api.py +++ b/cds/modules/deposit/api.py @@ -400,7 +400,6 @@ def _publish_edited(self): record = self._generate_smil_file(record.id, record, bucket) # dump after smil generation record["_files"] = record.files.dumps() - print(self.files.dumps()) bucket.locked = True return record diff --git a/setup.cfg b/setup.cfg index 395b4c750..60a9e5b2b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -236,4 +236,4 @@ ignore = [tool:pytest] # addopts = --black --isort --pydocstyle --cov=cds_videos --cov-report=term-missing -testpaths = tests unit +testpaths = tests/unit