Add background option to backup APIs (#4802)

* Add background option to backup APIs

* Fix decorator tests

* Working error handling, initial test cases

* Change to schedule_job and always return job id

* Add tests

* Reorder call at/later args

* Validation errors return immediately in background

* None is invalid option for background

* Must pop the background option from body

supervisor/api/__init__.py

@@ -219,6 +219,8 @@ def _register_jobs(self) -> None:
                 web.get("/jobs/info", api_jobs.info),
                 web.post("/jobs/options", api_jobs.options),
                 web.post("/jobs/reset", api_jobs.reset),
+                web.get("/jobs/{uuid}", api_jobs.job_info),
+                web.delete("/jobs/{uuid}", api_jobs.remove_job),
             ]
         )
 

supervisor/api/backups.py

@@ -1,5 +1,6 @@
 """Backups RESTful API."""
 import asyncio
+from collections.abc import Callable
 import errno
 import logging
 from pathlib import Path
@@ -11,6 +12,7 @@
 from aiohttp.hdrs import CONTENT_DISPOSITION
 import voluptuous as vol
 
+from ..backups.backup import Backup
 from ..backups.validate import ALL_FOLDERS, FOLDER_HOMEASSISTANT, days_until_stale
 from ..const import (
     ATTR_ADDONS,
@@ -33,12 +35,15 @@
     ATTR_TIMEOUT,
     ATTR_TYPE,
     ATTR_VERSION,
+    BusEvent,
+    CoreState,
 )
 from ..coresys import CoreSysAttributes
 from ..exceptions import APIError
+from ..jobs import JobSchedulerOptions
 from ..mounts.const import MountUsage
 from ..resolution.const import UnhealthyReason
-from .const import CONTENT_TYPE_TAR
+from .const import ATTR_BACKGROUND, ATTR_JOB_ID, CONTENT_TYPE_TAR
 from .utils import api_process, api_validate
 
 _LOGGER: logging.Logger = logging.getLogger(__name__)
@@ -50,24 +55,29 @@
 _ALL_FOLDERS = ALL_FOLDERS + [FOLDER_HOMEASSISTANT]
 
 # pylint: disable=no-value-for-parameter
-SCHEMA_RESTORE_PARTIAL = vol.Schema(
+SCHEMA_RESTORE_FULL = vol.Schema(
     {
         vol.Optional(ATTR_PASSWORD): vol.Maybe(str),
+        vol.Optional(ATTR_BACKGROUND, default=False): vol.Boolean(),
+    }
+)
+
+SCHEMA_RESTORE_PARTIAL = SCHEMA_RESTORE_FULL.extend(
+    {
         vol.Optional(ATTR_HOMEASSISTANT): vol.Boolean(),
         vol.Optional(ATTR_ADDONS): vol.All([str], vol.Unique()),
         vol.Optional(ATTR_FOLDERS): vol.All([vol.In(_ALL_FOLDERS)], vol.Unique()),
     }
 )
 
-SCHEMA_RESTORE_FULL = vol.Schema({vol.Optional(ATTR_PASSWORD): vol.Maybe(str)})
-
 SCHEMA_BACKUP_FULL = vol.Schema(
     {
         vol.Optional(ATTR_NAME): str,
         vol.Optional(ATTR_PASSWORD): vol.Maybe(str),
         vol.Optional(ATTR_COMPRESSED): vol.Maybe(vol.Boolean()),
         vol.Optional(ATTR_LOCATON): vol.Maybe(str),
         vol.Optional(ATTR_HOMEASSISTANT_EXCLUDE_DATABASE): vol.Boolean(),
+        vol.Optional(ATTR_BACKGROUND, default=False): vol.Boolean(),
     }
 )
 
@@ -204,46 +214,109 @@ def _location_to_mount(self, body: dict[str, Any]) -> dict[str, Any]:
 
         return body
 
+    async def _background_backup_task(
+        self, backup_method: Callable, *args, **kwargs
+    ) -> tuple[asyncio.Task, str]:
+        """Start backup task in  background and return task and job ID."""
+        event = asyncio.Event()
+        job, backup_task = self.sys_jobs.schedule_job(
+            backup_method, JobSchedulerOptions(), *args, **kwargs
+        )
+
+        async def release_on_freeze(new_state: CoreState):
+            if new_state == CoreState.FREEZE:
+                event.set()
+
+        # Wait for system to get into freeze state before returning
+        # If the backup fails validation it will raise before getting there
+        listener = self.sys_bus.register_event(
+            BusEvent.SUPERVISOR_STATE_CHANGE, release_on_freeze
+        )
+        try:
+            await asyncio.wait(
+                (
+                    backup_task,
+                    self.sys_create_task(event.wait()),
+                ),
+                return_when=asyncio.FIRST_COMPLETED,
+            )
+            return (backup_task, job.uuid)
+        finally:
+            self.sys_bus.remove_listener(listener)
+
     @api_process
     async def backup_full(self, request):
         """Create full backup."""
         body = await api_validate(SCHEMA_BACKUP_FULL, request)
-
-        backup = await asyncio.shield(
-            self.sys_backups.do_backup_full(**self._location_to_mount(body))
+        background = body.pop(ATTR_BACKGROUND)
+        backup_task, job_id = await self._background_backup_task(
+            self.sys_backups.do_backup_full, **self._location_to_mount(body)
         )
 
+        if background and not backup_task.done():
+            return {ATTR_JOB_ID: job_id}
+
+        backup: Backup = await backup_task
         if backup:
-            return {ATTR_SLUG: backup.slug}
-        return False
+            return {ATTR_JOB_ID: job_id, ATTR_SLUG: backup.slug}
+        raise APIError(
+            f"An error occurred while making backup, check job '{job_id}' or supervisor logs for details",
+            job_id=job_id,
+        )
 
     @api_process
     async def backup_partial(self, request):
         """Create a partial backup."""
         body = await api_validate(SCHEMA_BACKUP_PARTIAL, request)
-        backup = await asyncio.shield(
-            self.sys_backups.do_backup_partial(**self._location_to_mount(body))
+        background = body.pop(ATTR_BACKGROUND)
+        backup_task, job_id = await self._background_backup_task(
+            self.sys_backups.do_backup_partial, **self._location_to_mount(body)
         )
 
+        if background and not backup_task.done():
+            return {ATTR_JOB_ID: job_id}
+
+        backup: Backup = await backup_task
         if backup:
-            return {ATTR_SLUG: backup.slug}
-        return False
+            return {ATTR_JOB_ID: job_id, ATTR_SLUG: backup.slug}
+        raise APIError(
+            f"An error occurred while making backup, check job '{job_id}' or supervisor logs for details",
+            job_id=job_id,
+        )
 
     @api_process
     async def restore_full(self, request):
         """Full restore of a backup."""
         backup = self._extract_slug(request)
         body = await api_validate(SCHEMA_RESTORE_FULL, request)
+        background = body.pop(ATTR_BACKGROUND)
+        restore_task, job_id = await self._background_backup_task(
+            self.sys_backups.do_restore_full, backup, **body
+        )
 
-        return await asyncio.shield(self.sys_backups.do_restore_full(backup, **body))
+        if background and not restore_task.done() or await restore_task:
+            return {ATTR_JOB_ID: job_id}
+        raise APIError(
+            f"An error occurred during restore of {backup.slug}, check job '{job_id}' or supervisor logs for details",
+            job_id=job_id,
+        )
 
     @api_process
     async def restore_partial(self, request):
         """Partial restore a backup."""
         backup = self._extract_slug(request)
         body = await api_validate(SCHEMA_RESTORE_PARTIAL, request)
+        background = body.pop(ATTR_BACKGROUND)
+        restore_task, job_id = await self._background_backup_task(
+            self.sys_backups.do_restore_partial, backup, **body
+        )
 
-        return await asyncio.shield(self.sys_backups.do_restore_partial(backup, **body))
+        if background and not restore_task.done() or await restore_task:
+            return {ATTR_JOB_ID: job_id}
+        raise APIError(
+            f"An error occurred during restore of {backup.slug}, check job '{job_id}' or supervisor logs for details",
+            job_id=job_id,
+        )
 
     @api_process
     async def freeze(self, request):

supervisor/api/const.py

@@ -13,6 +13,7 @@
 ATTR_APPARMOR_VERSION = "apparmor_version"
 ATTR_ATTRIBUTES = "attributes"
 ATTR_AVAILABLE_UPDATES = "available_updates"
+ATTR_BACKGROUND = "background"
 ATTR_BOOT_TIMESTAMP = "boot_timestamp"
 ATTR_BOOTS = "boots"
 ATTR_BROADCAST_LLMNR = "broadcast_llmnr"
@@ -31,6 +32,7 @@
 ATTR_FALLBACK = "fallback"
 ATTR_FILESYSTEMS = "filesystems"
 ATTR_IDENTIFIERS = "identifiers"
+ATTR_JOB_ID = "job_id"
 ATTR_JOBS = "jobs"
 ATTR_LLMNR = "llmnr"
 ATTR_LLMNR_HOSTNAME = "llmnr_hostname"

supervisor/api/jobs.py

@@ -6,6 +6,7 @@
 import voluptuous as vol
 
 from ..coresys import CoreSysAttributes
+from ..exceptions import APIError
 from ..jobs import SupervisorJob
 from ..jobs.const import ATTR_IGNORE_CONDITIONS, JobCondition
 from .const import ATTR_JOBS
@@ -21,7 +22,7 @@
 class APIJobs(CoreSysAttributes):
     """Handle RESTful API for OS functions."""
 
-    def _list_jobs(self) -> list[dict[str, Any]]:
+    def _list_jobs(self, start: SupervisorJob | None = None) -> list[dict[str, Any]]:
         """Return current job tree."""
         jobs_by_parent: dict[str | None, list[SupervisorJob]] = {}
         for job in self.sys_jobs.jobs:
@@ -34,9 +35,11 @@ def _list_jobs(self) -> list[dict[str, Any]]:
                 jobs_by_parent[job.parent_id].append(job)
 
         job_list: list[dict[str, Any]] = []
-        queue: list[tuple[list[dict[str, Any]], SupervisorJob]] = [
-            (job_list, job) for job in jobs_by_parent.get(None, [])
-        ]
+        queue: list[tuple[list[dict[str, Any]], SupervisorJob]] = (
+            [(job_list, start)]
+            if start
+            else [(job_list, job) for job in jobs_by_parent.get(None, [])]
+        )
 
         while queue:
             (current_list, current_job) = queue.pop(0)
@@ -78,3 +81,19 @@ async def options(self, request: web.Request) -> None:
     async def reset(self, request: web.Request) -> None:
         """Reset options for JobManager."""
         self.sys_jobs.reset_data()
+
+    @api_process
+    async def job_info(self, request: web.Request) -> dict[str, Any]:
+        """Get details of a job by ID."""
+        job = self.sys_jobs.get_job(request.match_info.get("uuid"))
+        return self._list_jobs(job)[0]
+
+    @api_process
+    async def remove_job(self, request: web.Request) -> None:
+        """Remove a completed job."""
+        job = self.sys_jobs.get_job(request.match_info.get("uuid"))
+
+        if not job.done:
+            raise APIError(f"Job {job.uuid} is not done!")
+
+        self.sys_jobs.remove_job(job)

supervisor/api/utils.py

@@ -13,6 +13,7 @@
     HEADER_TOKEN,
     HEADER_TOKEN_OLD,
     JSON_DATA,
+    JSON_JOB_ID,
     JSON_MESSAGE,
     JSON_RESULT,
     REQUEST_FROM,
@@ -124,11 +125,15 @@ def api_return_error(
         if check_exception_chain(error, DockerAPIError):
             message = format_message(message)
 
+    result = {
+        JSON_RESULT: RESULT_ERROR,
+        JSON_MESSAGE: message or "Unknown error, see supervisor",
+    }
+    if isinstance(error, APIError) and error.job_id:
+        result[JSON_JOB_ID] = error.job_id
+
     return web.json_response(
-        {
-            JSON_RESULT: RESULT_ERROR,
-            JSON_MESSAGE: message or "Unknown error, see supervisor",
-        },
+        result,
         status=400,
         dumps=json_dumps,
     )