Merge pull request #53 from PermanentOrg/thumbnail_endpoint_optimizat…

…ions Update folder thumbnail recalculation endpoint to handle smaller batches
PermanentOrg · Aug 11, 2023 · a2cb348 · a2cb348
2 parents 63196b4 + dce7ef6
commit a2cb348
Show file tree

Hide file tree

Showing 8 changed files with 180 additions and 14 deletions.
diff --git a/API.md b/API.md
@@ -300,14 +300,16 @@ These endpoints are all authenticated with admin authentication tokens, generate
 [documentation](https://permanent.atlassian.net/wiki/spaces/EN/pages/2072576001/Trigger+Admin+Directives)
 
 ### POST `/admin/folder/recalculate_thumbnails`
+Queues thumbnail generation tasks for all non-root folders created between `beginTimestamp` and `endTimestamp`.
 
 - Headers: Authorization: Bearer \<JWT from FusionAuth>
 
 - Request
 
 ```
 {
-    cutoffTimestamp: string (date, iso format)
+    beginTimestamp: string (date, iso format)
+    endTimestamp: string (date, iso format)
 }
 ```
 

diff --git a/src/admin/controller.ts b/src/admin/controller.ts
@@ -13,7 +13,8 @@ adminController.post(
     try {
       validateRecalculateFolderThumbnailsRequest(req.body);
       const results = await adminService.recalculateFolderThumbnails(
-        req.body.cutoffTimestamp
+        req.body.beginTimestamp,
+        req.body.endTimestamp
       );
       if (results.failedFolders.length > 0) {
         res.status(500).json(results);

diff --git a/src/admin/queries/get_folders_created_before_timestamp.sql b/src/admin/queries/get_folders_created_before_timestamp.sql
@@ -4,4 +4,5 @@ SELECT
 FROM
   folder
 WHERE
-  createdDT < :cutoffTimestamp
+  createdDT BETWEEN :beginTimestamp AND :endTimestamp
+  AND (type IS NULL OR type NOT LIKE '%root%');
diff --git a/src/admin/service.test.ts b/src/admin/service.test.ts
@@ -35,8 +35,11 @@ describe("recalculateFolderThumbnails", () => {
   test("should send messages for folders created before the cutoff", async () => {
     jest
       .spyOn(publisherClient, "batchPublishMessages")
-      .mockResolvedValueOnce({ failedMessages: [], messagesSent: 5 });
-    const result = await adminService.recalculateFolderThumbnails(new Date());
+      .mockResolvedValueOnce({ failedMessages: [], messagesSent: 4 });
+    const result = await adminService.recalculateFolderThumbnails(
+      new Date(new Date().setDate(new Date().getDate() - 1)),
+      new Date()
+    );
     expect(
       (
         (
@@ -46,16 +49,19 @@ describe("recalculateFolderThumbnails", () => {
           ]
         )[1] as unknown as Message[]
       ).length
-    ).toBe(5);
-    expect(result).toEqual({ failedFolders: [], messagesSent: 5 });
+    ).toBe(4);
+    expect(result).toEqual({ failedFolders: [], messagesSent: 4 });
   });
 
   test("should throw internal server error if database call fails", async () => {
     let error = null;
     const testError = new Error("out of cheese - redo from start");
     jest.spyOn(db, "sql").mockRejectedValueOnce(testError);
     try {
-      await adminService.recalculateFolderThumbnails(new Date());
+      await adminService.recalculateFolderThumbnails(
+        new Date(new Date().setDate(new Date().getDate() - 1)),
+        new Date()
+      );
     } catch (err) {
       error = err;
     } finally {
@@ -71,7 +77,10 @@ describe("recalculateFolderThumbnails", () => {
       .spyOn(publisherClient, "batchPublishMessages")
       .mockRejectedValueOnce(testError);
     try {
-      await adminService.recalculateFolderThumbnails(new Date());
+      await adminService.recalculateFolderThumbnails(
+        new Date(new Date().setDate(new Date().getDate() - 1)),
+        new Date()
+      );
     } catch (err) {
       error = err;
     } finally {

diff --git a/src/admin/service.ts b/src/admin/service.ts
@@ -5,11 +5,13 @@ import { publisherClient, lowPriorityTopicArn } from "../publisher_client";
 import { logger } from "../log";
 
 const recalculateFolderThumbnails = async (
-  cutoffTimestamp: Date
+  beginTimestamp: Date,
+  endTimestamp: Date
 ): Promise<{ messagesSent: number; failedFolders: string[] }> => {
   const folderResult = await db
     .sql<Folder>("admin.queries.get_folders_created_before_timestamp", {
-      cutoffTimestamp,
+      beginTimestamp,
+      endTimestamp,
     })
     .catch((err) => {
       logger.error(err);

diff --git a/src/admin/validators.test.ts b/src/admin/validators.test.ts
@@ -0,0 +1,150 @@
+import { validateRecalculateFolderThumbnailsRequest } from "./validators";
+
+describe("validateRecalculateFolderThumbnailsRequest", () => {
+  test("should not error if the request is valid", () => {
+    let error = null;
+    try {
+      validateRecalculateFolderThumbnailsRequest({
+        emailFromAuthToken: "test@permanent.org",
+        beginTimestamp: "2023-07-30",
+        endTimestamp: "2023-07-31",
+      });
+    } catch (err) {
+      error = err;
+    } finally {
+      expect(error).toBeNull();
+    }
+  });
+
+  test("should error if emailFromAuthToken is missing", () => {
+    let error = null;
+    try {
+      validateRecalculateFolderThumbnailsRequest({
+        beginTimestamp: "2023-07-30",
+        endTimestamp: "2023-07-31",
+      });
+    } catch (err) {
+      error = err;
+    } finally {
+      expect(error).not.toBeNull();
+    }
+  });
+
+  test("should error if emailFromAuthToken is wrong type", () => {
+    let error = null;
+    try {
+      validateRecalculateFolderThumbnailsRequest({
+        emailFromAuthToken: 123,
+        beginTimestamp: "2023-07-30",
+        endTimestamp: "2023-07-31",
+      });
+    } catch (err) {
+      error = err;
+    } finally {
+      expect(error).not.toBeNull();
+    }
+  });
+
+  test("should error if emailFromAuthToken is wrong format", () => {
+    let error = null;
+    try {
+      validateRecalculateFolderThumbnailsRequest({
+        emailFromAuthToken: "not_an_email",
+        beginTimestamp: "2023-07-30",
+        endTimestamp: "2023-07-31",
+      });
+    } catch (err) {
+      error = err;
+    } finally {
+      expect(error).not.toBeNull();
+    }
+  });
+
+  test("should error if beginTimestamp is missing", () => {
+    let error = null;
+    try {
+      validateRecalculateFolderThumbnailsRequest({
+        emailFromAuthToken: "test@permanent.org",
+        endTimestamp: "2023-07-31",
+      });
+    } catch (err) {
+      error = err;
+    } finally {
+      expect(error).not.toBeNull();
+    }
+  });
+
+  test("should error if beginTimestamp is wrong type", () => {
+    let error = null;
+    try {
+      validateRecalculateFolderThumbnailsRequest({
+        emailFromAuthToken: "test@permanent.org",
+        beginTimestamp: "not_a_date",
+        endTimestamp: "2023-07-31",
+      });
+    } catch (err) {
+      error = err;
+    } finally {
+      expect(error).not.toBeNull();
+    }
+  });
+
+  test("should error if beginTimestamp is wrong format", () => {
+    let error = null;
+    try {
+      validateRecalculateFolderThumbnailsRequest({
+        emailFromAuthToken: "test@permanent.org",
+        beginTimestamp: "07/31/23",
+        endTimestamp: "2023-07-31",
+      });
+    } catch (err) {
+      error = err;
+    } finally {
+      expect(error).not.toBeNull();
+    }
+  });
+
+  test("should error if endTimestamp is missing", () => {
+    let error = null;
+    try {
+      validateRecalculateFolderThumbnailsRequest({
+        emailFromAuthToken: "test@permanent.org",
+        beginTimestamp: "2023-07-30",
+      });
+    } catch (err) {
+      error = err;
+    } finally {
+      expect(error).not.toBeNull();
+    }
+  });
+
+  test("should error if endTimestamp is wrong type", () => {
+    let error = null;
+    try {
+      validateRecalculateFolderThumbnailsRequest({
+        emailFromAuthToken: "test@permanent.org",
+        beginTimestamp: "2023-07-30",
+        endTimestamp: "not_a_date",
+      });
+    } catch (err) {
+      error = err;
+    } finally {
+      expect(error).not.toBeNull();
+    }
+  });
+
+  test("should error if endTimestamp is wrong format", () => {
+    let error = null;
+    try {
+      validateRecalculateFolderThumbnailsRequest({
+        emailFromAuthToken: "test@permanent.org",
+        beginTimestamp: "2023-07-30",
+        endTimestamp: "07/31/23",
+      });
+    } catch (err) {
+      error = err;
+    } finally {
+      expect(error).not.toBeNull();
+    }
+  });
+});
diff --git a/src/admin/validators.ts b/src/admin/validators.ts
@@ -2,11 +2,12 @@ import Joi from "joi";
 
 export function validateRecalculateFolderThumbnailsRequest(
   data: unknown
-): asserts data is { cutoffTimestamp: Date } {
+): asserts data is { beginTimestamp: Date; endTimestamp: Date } {
   const validation = Joi.object()
     .keys({
       emailFromAuthToken: Joi.string().email().required(),
-      cutoffTimestamp: Joi.date().iso().required(),
+      beginTimestamp: Joi.date().iso().required(),
+      endTimestamp: Joi.date().iso().required(),
     })
     .validate(data);
   if (validation.error) {

diff --git a/src/fixtures/create_test_folders.sql b/src/fixtures/create_test_folders.sql
@@ -1,7 +1,7 @@
 INSERT INTO 
   folder (folderId, archiveId, publicDt, displayName, status, createdDt)
 VALUES
-  (1, 1, '2023-06-21', 'Public Folder', 'status.generic.ok', CURRENT_TIMESTAMP),
+  (1, 1, '2023-06-21', 'Public Folder', 'status.generic.ok', CURRENT_TIMESTAMP - '2 days'::interval),
   (2, 1, NULL, 'Private Folder', 'status.generic.ok', CURRENT_TIMESTAMP),
   (3, 1, CURRENT_TIMESTAMP + '1 day'::interval, 'Future Public Folder', 'status.generic.ok', CURRENT_TIMESTAMP),
   (4, 1, '2023-06-21', 'Deleted Folder', 'status.generic.deleted', CURRENT_TIMESTAMP),