working POC of displaying a loading overlay while waiting for the man…

…ifest to create in RCRAInfo

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "haztrak",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "private": true,
   "scripts": {
     "start": "vite",

client/src/components/Manifest/ManifestForm.tsx

@@ -2,7 +2,7 @@ import { ErrorMessage } from '@hookform/error-message';
 import { zodResolver } from '@hookform/resolvers/zod';
 import { AdditionalInfoForm } from 'components/AdditionalInfo/AdditionalInfoForm';
 import { HtButton, HtCard, HtForm, InfoIconTooltip } from 'components/Ht';
-import { UpdateRcra } from 'components/Manifest/rcraLoading/UpdateRcra';
+import { UpdateRcra } from 'components/Manifest/UpdateRcra/UpdateRcra';
 import { WasteLine } from 'components/Manifest/WasteLine/wasteLineSchema';
 import { RcraSiteDetails } from 'components/RcraSite';
 import React, { createContext, useState } from 'react';

client/src/components/Manifest/rcraLoading/UpdateRcra.tsx → client/src/components/Manifest/UpdateRcra/UpdateRcra.tsx

@@ -15,12 +15,13 @@ export function UpdateRcra({ taskId }: UpdateRcraProps) {
     pollingInterval: 3000,
   });
 
-  if (data?.result) {
-    const resp = JSON.parse(data?.result);
+  if (data?.status === 'SUCCESS') {
+    const resp = data?.result;
     return <Navigate to={`/manifest/${resp.manifestTrackingNumber}/view`} />;
   }
 
-  if (error) {
+  // @ts-ignore
+  if (error && error.status !== 404) {
     return (
       <ToastContainer position="top-end" style={{ zIndex: 1 }} className={'p-3'}>
         <Toast bg="danger" onClose={() => setShowToast(false)} show={showToast}>
@@ -35,12 +36,10 @@ export function UpdateRcra({ taskId }: UpdateRcraProps) {
       </ToastContainer>
     );
   }
-  // if (isLoading || !data || !results) {
-  if (isLoading) {
-    return (
-      <div className="overlay-spinner">
-        <HtSpinner className="text-light" size="5x" />
-      </div>
-    );
-  }
+
+  return (
+    <div className="overlay-spinner">
+      <HtSpinner className="text-light" size="5x" />
+    </div>
+  );
 }

server/.gitignore

@@ -1 +1,2 @@
 cache/
+test_db

server/apps/core/serializers.py

@@ -1,3 +1,5 @@
+from collections import OrderedDict
+
 from django_celery_results.models import TaskResult
 from rest_framework import serializers
 from rest_framework.serializers import ModelSerializer
@@ -143,4 +145,9 @@ class TaskStatusSerializer(serializers.Serializer):
     )
     result = serializers.JSONField(
         required=False,
+        allow_null=True,
     )
+
+    def to_representation(self, instance):
+        result = super().to_representation(instance)
+        return OrderedDict([(key, result[key]) for key in result if result[key] is not None])

server/apps/core/services/__init__.py

@@ -1 +1,2 @@
 from .rcrainfo_service import RcrainfoService
+from .task_service import TaskService

server/apps/core/services/task_service.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Optional
 
 from django.core.cache import CacheKeyWarning, cache
 from django_celery_results.models import TaskResult
@@ -16,8 +17,11 @@ class TaskService:
     Service class for interacting with the Task model layer and celery tasks.
     """
 
-    def __init__(self):
-        pass
+    def __init__(self, task_id, task_name, status="PENDING", result=None):
+        self.task_id = task_id
+        self.task_name = task_name
+        self.status = status
+        self.result: dict | None = result
 
     @classmethod
     def get_task_status(cls, task_id) -> ReturnDict:
@@ -72,3 +76,29 @@ def launch_example_task():
             return task.id
         except KeyError:
             return None
+
+    def update_task_status(self, status: str, results: Optional = None) -> object | None:
+        """
+        Updates the status of a long-running celery task in our key-value store
+        returns an error or None
+        """
+        if results:
+            self.result = results
+        try:
+            task_serializer = TaskStatusSerializer(
+                data={
+                    "taskId": self.task_id,
+                    "status": status,
+                    "taskName": self.task_name,
+                    "result": self.result,
+                }
+            )
+            if task_serializer.is_valid():
+                logger.debug(f"task_serializer.data: {task_serializer.data}")
+                cache.set(self.task_id, task_serializer.data)
+                return task_serializer.data
+            logger.error(f"Could not serialize task status: {task_serializer.errors}")
+            return None
+        except CacheKeyWarning as e:
+            logger.error(e)
+            return e

server/apps/core/tests/test_task_view.py → server/apps/core/tests/test_tasks.py

@@ -5,6 +5,7 @@
 from rest_framework.response import Response
 from rest_framework.test import APIRequestFactory, force_authenticate
 
+from apps.core.services.task_service import TaskService
 from apps.core.views import TaskStatusView
 
 
@@ -70,3 +71,19 @@ def test_returns_db_results(self, factory, user, cache_factory):
         # Assert
         assert response.status_code == status.HTTP_200_OK
         assert response.data["taskId"] == self.mock_task_id
+
+
+class TestTaskService:
+    mock_task_id = "mock_task_id"
+    mock_task_name = "mock_task_name"
+
+    def test_returns_cached_status(self, cache_factory):
+        # Arrange
+        cache_factory("test_returns_404")
+        haztrak_task = TaskService(task_id=self.mock_task_id, task_name=self.mock_task_name)
+        haztrak_task.update_task_status(status="STARTED")
+        # Act
+        cache_data = cache.get(self.mock_task_id)
+        # Assert
+        assert cache_data["taskName"] == self.mock_task_name
+        assert cache_data["status"] == "STARTED"

server/apps/trak/services/manifest_service.py

@@ -160,7 +160,7 @@ def create_rcra_manifest(self, *, manifest: dict) -> RcrainfoResponse:
         :param manifest: Dict
         :return:
         """
-        logger.info(f"create rcra manifest with arguments: {manifest}")
+        logger.debug(f"start create rcra manifest with arguments {manifest}")
         create_resp: RcrainfoResponse = self.rcrainfo.save_manifest(manifest)
         try:
             if create_resp.ok:
@@ -181,9 +181,7 @@ def _filter_mtn(signature: QuickerSign) -> dict[str, list[str]]:
         results = {"success": [], "error": []}
         site_filter = Manifest.objects.get_handler_query(signature.site_id, signature.site_type)
         existing_mtn = Manifest.objects.existing_mtn(site_filter, mtn=signature.mtn)
-        # get our list of valid MTN
         results["success"] = [manifest.mtn for manifest in existing_mtn]
-        # append any MTN, passed as an argument, not found in the DB to the error results
         results["error"].extend(list(set(signature.mtn).difference(set(results["success"]))))
         logger.warning(f"MTN not found or site not listed as site type {results['error']}")
         return results

server/apps/trak/tasks/manifest_task.py

@@ -18,16 +18,21 @@ def pull_manifest(self: Task, *, mtn: List[str], username: str) -> dict:
     This task initiates a call to the ManifestService to pull a manifest by MTN
     """
 
+    from apps.core.services import TaskService
     from apps.trak.services import ManifestService
 
     logger.debug(f"start task {self.name}, manifest {mtn}")
+    task_status = TaskService(task_id=self.request.id, task_name=self.name, status="STARTED")
     try:
         manifest_service = ManifestService(username=username)
         results = manifest_service.pull_manifests(tracking_numbers=mtn)
+        task_status.update_task_status(status="SUCCESS", results=results)
         return results
     except (ConnectionError, TimeoutError):
+        task_status.update_task_status(status="FAILURE")
         raise Reject()
     except Exception as exc:
+        task_status.update_task_status(status="FAILURE")
         self.update_state(state=states.FAILURE, meta=f"unknown error: {exc}")
         raise Ignore()
 
@@ -87,26 +92,28 @@ def sync_site_manifests(self, *, site_id: str, username: str):
         raise Ignore()
 
 
-# create_rcra_manifest
 @shared_task(name="create rcra manifests", bind=True)
 def create_rcra_manifest(self, *, manifest: dict, username: str):
     """
     asynchronous task to use the RCRAInfo web services to create an electronic (RCRA) manifest
     it accepts a Python dict of the manifest data to be submitted as JSON, and the username of the
     user who is creating the manifest
     """
+    from apps.core.services import TaskService
     from apps.trak.services import ManifestService
 
     logger.info(f"start task: {self.name}")
+    task_status = TaskService(task_id=self.request.id, task_name=self.name, status="STARTED")
     try:
-        logger.debug(f"creating manifest: {manifest}")
         manifest_service = ManifestService(username=username)
         resp: RcrainfoResponse = manifest_service.create_rcra_manifest(manifest=manifest)
         if resp.ok:
-            logger.info(f"successfully created manifest: {manifest}")
+            task_status.update_task_status(status="SUCCESS", results=resp.json())
             return resp.json()
         logger.error(f"failed to create manifest ({manifest}): {resp.json()}")
+        task_status.update_task_status(status="FAILURE", results=resp.json())
         return resp.json()
     except Exception as exc:
         logger.error("error: ", exc)
+        task_status.update_task_status(status="FAILURE", results=str(exc))
         return {"error": f"Internal Error: {exc}"}

server/apps/trak/views/manifest_view.py

@@ -2,13 +2,15 @@
 import logging
 
 from celery.exceptions import TaskError
+from celery.result import AsyncResult
 from django.db.models import Q
 from drf_spectacular.utils import extend_schema
 from rest_framework import status, viewsets
 from rest_framework.generics import GenericAPIView, ListAPIView
 from rest_framework.request import Request
 from rest_framework.response import Response
 
+from apps.core.services import TaskService
 from apps.sites.models import Site
 from apps.trak.models import Manifest
 from apps.trak.serializers import ManifestSerializer, MtnSerializer
@@ -69,7 +71,6 @@ def get_queryset(self):
         else:
             sites = [i.rcra_site.epa_id for i in Site.objects.filter(rcra_site__epa_id=epa_id)]
 
-        logger.info(sites)
         return Manifest.objects.filter(
             Q(generator__rcra_site__epa_id__in=sites) | Q(tsdf__rcra_site__epa_id__in=sites)
         )
@@ -141,9 +142,11 @@ def post(self, request: Request) -> Response:
                 f"valid manifest data submitted for creation in RCRAInfo: "
                 f"{datetime.datetime.utcnow()}"
             )
-            task = create_rcra_manifest.delay(
+            task: AsyncResult = create_rcra_manifest.delay(
                 manifest=manifest_serializer.data, username=str(request.user)
             )
+            task_status = TaskService(task_id=task.id, task_name=task.name, status="STARTED")
+            task_status.update_task_status(status="PENDING")
             return self.response(data={"taskId": task.id}, status=status.HTTP_201_CREATED)
         else:
             logger.error("manifest_serializer errors: ", manifest_serializer.errors)

server/haztrak/settings.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 
 # Globals
-HAZTRAK_VERSION = "0.4.0"
+HAZTRAK_VERSION = "0.7.0"
 
 # Environment variable mappings
 HOST_ENV = "HT_HOST"

server/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "haztrak"
-version = "0.3.0"
+version = "0.7.0"
 description = "An open-source web app illustrating how waste management software can interface with RCRAInfo to track hazardous waste"
 readme = "README.md"
 authors = [