Video downloads have many more options. Downloads are migrated.

Storing video settings in session storage to help user start downloads consistently.

Applying channel tag name to new channels when download videos or channels.

Initializing map tiles when importing and resetting map.

.circleci/config.yml

@@ -191,6 +191,31 @@ jobs:
       - run:
           name: Stop React App
           command: pkill -f npm
+  react-app-cypress-components:
+    docker:
+      - image: cypress/browsers:node-20.18.0-chrome-130.0.6723.69-1-ff-131.0.3-edge-130.0.2849.52-1
+    resource_class: small
+    parallelism: 3
+    steps:
+      - checkout
+      - restore_cache:
+          key: dependency-cache-{{ checksum "app/package-lock.json" }}
+      - run: cd app && npm install --legacy-peer-deps
+      - run:
+          name: Run Cypress component tests
+          command: |
+            cd app
+            # Run tests for Chrome
+            echo "Running tests in Chrome"
+            npx cypress run --component --browser chrome
+
+            # Run tests for Firefox
+            echo "Running tests in Firefox"
+            npx cypress run --component --browser firefox
+
+            # Run tests for Edge
+            echo "Running tests in Edge"
+            npx cypress run --component --browser edge
 
 workflows:
   wrolpi-api-tests:
@@ -206,3 +231,4 @@ workflows:
       - app-tests-22
       - react-app-build
       - react-app-start
+      - react-app-cypress-components

alembic/versions/170d1be52bc7_download_destination.py

@@ -0,0 +1,35 @@
+"""Create Download.destination column.
+
+Revision ID: 170d1be52bc7
+Revises: 9ec4c765ef8d
+Create Date: 2024-11-02 11:16:15.297850
+
+"""
+import os
+
+from alembic import op
+from sqlalchemy.orm import Session
+
+# revision identifiers, used by Alembic.
+revision = '170d1be52bc7'
+down_revision = '9ec4c765ef8d'
+branch_labels = None
+depends_on = None
+
+DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
+
+
+def upgrade():
+    bind = op.get_bind()
+    session = Session(bind=bind)
+
+    session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS destination TEXT')
+    session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS tag_names TEXT[]')
+
+
+def downgrade():
+    bind = op.get_bind()
+    session = Session(bind=bind)
+
+    session.execute('ALTER TABLE download DROP COLUMN IF EXISTS tag_names')
+    session.execute('ALTER TABLE download DROP COLUMN IF EXISTS destination')

alembic/versions/1f2325523525_download_destination_migrate.py

@@ -0,0 +1,137 @@
+"""Create `Download.destination` and `Download.tag_names`, migrate from `Download.settings`.
+
+Revision ID: 1f2325523525
+Revises: 170d1be52bc7
+Create Date: 2024-11-02 11:20:45.028865
+
+"""
+import os
+from copy import copy
+
+from alembic import op
+from sqlalchemy import Column, Integer, Text, String, ARRAY
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import Session
+
+from wrolpi.common import ModelHelper
+from wrolpi.dates import TZDateTime
+from wrolpi.downloader import get_download_manager_config, DownloadStatus
+
+# revision identifiers, used by Alembic.
+revision = '1f2325523525'
+down_revision = '170d1be52bc7'
+branch_labels = None
+depends_on = None
+
+DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
+
+Base = declarative_base()
+
+
+class MDownload(ModelHelper, Base):
+    __tablename__ = 'download'
+    id = Column(Integer, primary_key=True)
+    url = Column(String, nullable=False, unique=True)
+    attempts = Column(Integer, default=0)
+    destination = Column(Text)
+    downloader = Column(Text)
+    sub_downloader = Column(Text)
+    error = Column(Text)
+    frequency = Column(Integer)
+    info_json = Column(JSONB)
+    last_successful_download = Column(TZDateTime)
+    location = Column(Text)
+    next_download = Column(TZDateTime)
+    settings = Column(JSONB)
+    status = Column(String, default=DownloadStatus.new)
+    tag_names = Column(ARRAY(Text))
+
+
+def upgrade():
+    bind = op.get_bind()
+    session = Session(bind=bind)
+
+    for download in session.query(MDownload):
+        if not download.settings:
+            continue
+        # Move `download.settings['destination']` to `download.destination`, if any.
+        settings = copy(download.settings) if download.settings else dict()
+        destination = settings.pop('destination', None)
+        tag_names = settings.pop('tag_names', None)
+        if isinstance(excluded_urls := settings.get('excluded_urls'), list):
+            settings['excluded_urls'] = ','.join(excluded_urls)
+        download.destination = destination or None
+        download.tag_names = tag_names or None
+        download.settings = settings
+
+    # This will migrate the config as-is at the time that this migration was written.  This code is probably not
+    # safe to reuse in the future!
+
+    # Read config file directly.
+    try:
+        config = get_download_manager_config()
+        config._config.update(config.read_config_file())
+    except FileNotFoundError:
+        # Config does not exist, probably testing, or a new WROLPi.
+        return
+
+    new_downloads = []
+    for download in session.query(MDownload).order_by(MDownload.url):
+        if download.last_successful_download and not download.frequency:
+            # This once-download has completed, do not save it.
+            continue
+        new_download = dict(
+            destination=download.destination,
+            downloader=download.downloader,
+            frequency=download.frequency,
+            last_successful_download=download.last_successful_download,
+            next_download=download.next_download,
+            settings=download.settings,
+            status=download.status,
+            sub_downloader=download.sub_downloader,
+            url=download.url,
+        )
+        new_downloads.append(new_download)
+
+    # Write directly to file, bypassing usual checks.
+    config._config['downloads'] = new_downloads
+    config.write_config_data(config._config, config.get_file())
+
+
+def downgrade():
+    bind = op.get_bind()
+    session = Session(bind=bind)
+
+    for download in session.query(MDownload):
+        if download.destination:
+            settings = copy(download.settings) if download.settings else dict()
+            settings['destination'] = download.destination or settings.get('destination')
+            settings['tag_names'] = download.tag_names or settings.get('tag_names')
+            if isinstance(excluded_urls := settings.get('excluded_urls'), str):
+                settings['excluded_urls'] = excluded_urls.split(',')
+            download.settings = settings
+
+    try:
+        config = get_download_manager_config()
+        config._config.update(config.read_config_file())
+    except FileNotFoundError:
+        # Config does not exist, probably testing, or a new WROLPi.
+        return
+
+    new_downloads = []
+    for download in session.query(MDownload).order_by(MDownload.url):
+        new_download = dict(
+            downloader=download.downloader,
+            frequency=download.frequency,
+            last_successful_download=download.last_successful_download,
+            next_download=download.next_download,
+            settings=download.settings,
+            status=download.status,
+            sub_downloader=download.sub_downloader,
+            url=download.url,
+        )
+        new_downloads.append(new_download)
+
+    config._config['downloads'] = new_downloads
+    config.write_config_data(config._config, config.get_file())

alembic/versions/8d0d81bc9c34_channel_channel_downloads.py

@@ -9,12 +9,16 @@
 import pathlib
 
 from alembic import op
-from sqlalchemy import Column, Integer, String, Boolean, JSON, Date
+from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, Text, ForeignKey
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import Session, deferred
+from sqlalchemy.orm import Session, deferred, relationship
+from sqlalchemy.orm.collections import InstrumentedList
 
 from modules.videos.lib import link_channel_and_downloads
 from wrolpi.common import ModelHelper
+from wrolpi.dates import TZDateTime
+from wrolpi.downloader import DownloadStatus
 from wrolpi.media_path import MediaPathType
 
 # revision identifiers, used by Alembic.
@@ -43,6 +47,8 @@ class MChannel(ModelHelper, Base):
     info_json = deferred(Column(JSON))
     info_date = Column(Date)
 
+    downloads: InstrumentedList = relationship('MDownload', primaryjoin='MDownload.channel_id==MChannel.id')
+
     @staticmethod
     def get_by_url(url: str, session: Session = None):
         if not url:
@@ -55,14 +61,36 @@ def get_rss_url(self) -> str | None:
             return f'https://www.youtube.com/feeds/videos.xml?channel_id={self.source_id}'
 
 
+# The `Download` model at the time of this migration.
+class MDownload(ModelHelper, Base):
+    __tablename__ = 'download'
+    id = Column(Integer, primary_key=True)
+    url = Column(String, nullable=False, unique=True)
+
+    attempts = Column(Integer, default=0)
+    downloader = Column(Text)
+    sub_downloader = Column(Text)
+    error = Column(Text)
+    frequency = Column(Integer)
+    info_json = Column(JSONB)
+    last_successful_download = Column(TZDateTime)
+    location = Column(Text)
+    next_download = Column(TZDateTime)
+    settings = Column(JSONB)
+    status = Column(String, default=DownloadStatus.new)
+
+    channel_id = Column(Integer, ForeignKey('channel.id'))
+    channel = relationship('MChannel', primaryjoin='MDownload.channel_id==MChannel.id', back_populates='downloads')
+
+
 def upgrade():
     bind = op.get_bind()
     session = Session(bind=bind)
 
     session.execute('ALTER TABLE download ADD CONSTRAINT download_url_unique UNIQUE (url)')
     session.execute('ALTER TABLE download ADD COLUMN channel_id INTEGER REFERENCES channel(id)')
 
-    link_channel_and_downloads(session, MChannel)
+    link_channel_and_downloads(session, MChannel, MDownload)
 
     session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS match_regex')
     session.execute('ALTER TABLE channel DROP COLUMN IF EXISTS download_frequency')

app/cypress.config.js

@@ -0,0 +1,10 @@
+const { defineConfig } = require("cypress");
+
+module.exports = defineConfig({
+  component: {
+    devServer: {
+      framework: "create-react-app",
+      bundler: "webpack",
+    },
+  },
+});

app/cypress/fixtures/example.json

@@ -0,0 +1,5 @@
+{
+  "name": "Using fixtures to represent data",
+  "email": "hello@cypress.io",
+  "body": "Fixtures are a great way to mock data for responses to routes"
+}

app/cypress/support/commands.js

@@ -0,0 +1,25 @@
+// ***********************************************
+// This example commands.js shows you how to
+// create various custom commands and overwrite
+// existing commands.
+//
+// For more comprehensive examples of custom
+// commands please read more here:
+// https://on.cypress.io/custom-commands
+// ***********************************************
+//
+//
+// -- This is a parent command --
+// Cypress.Commands.add('login', (email, password) => { ... })
+//
+//
+// -- This is a child command --
+// Cypress.Commands.add('drag', { prevSubject: 'element'}, (subject, options) => { ... })
+//
+//
+// -- This is a dual command --
+// Cypress.Commands.add('dismiss', { prevSubject: 'optional'}, (subject, options) => { ... })
+//
+//
+// -- This will overwrite an existing command --
+// Cypress.Commands.overwrite('visit', (originalFn, url, options) => { ... })

app/cypress/support/component-index.html

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width,initial-scale=1.0">
+    <title>Components App</title>
+  </head>
+  <body>
+    <div data-cy-root></div>
+  </body>
+</html>

app/cypress/support/component.js

@@ -0,0 +1,27 @@
+// ***********************************************************
+// This example support/component.js is processed and
+// loaded automatically before your test files.
+//
+// This is a great place to put global configuration and
+// behavior that modifies Cypress.
+//
+// You can change the location of this file or turn off
+// automatically serving support files with the
+// 'supportFile' configuration option.
+//
+// You can read more here:
+// https://on.cypress.io/configuration
+// ***********************************************************
+
+// Import commands.js using ES2015 syntax:
+import './commands'
+
+// Alternatively you can use CommonJS syntax:
+// require('./commands')
+
+import { mount } from 'cypress/react18'
+
+Cypress.Commands.add('mount', mount)
+
+// Example use:
+// cy.mount(<MyComponent />)