From dce7ef6f134584adecbaaaef8ec761bd38f5c465 Mon Sep 17 00:00:00 2001 From: Liam Lloyd Date: Tue, 8 Aug 2023 17:27:28 -0700 Subject: [PATCH] Update folder thumbnail recalculation endpoint to handle smaller batches Upon testing folder thumbnail recalculation in our development environment, it became clear that it would take several hours to process all our folders. Since this could cause customer actions to get stuck in the queue behind all those thumbnail generation tasks, this is undesirable. This commit updates the endpoint that triggers thumbnail regeneration to take two timestamps and process only folders created between those two times. This will allow us to regenerate thumbnails in smaller batches and avoid clogging up the queues for long stretches. The commit also updates the endpoint to ignore root folders, for which thumbnails aren't used anyway (the thumbnail generation task skips root folders). --- API.md | 4 +- src/admin/controller.ts | 3 +- .../get_folders_created_before_timestamp.sql | 3 +- src/admin/service.test.ts | 21 ++- src/admin/service.ts | 6 +- src/admin/validators.test.ts | 150 ++++++++++++++++++ src/admin/validators.ts | 5 +- src/fixtures/create_test_folders.sql | 2 +- 8 files changed, 180 insertions(+), 14 deletions(-) create mode 100644 src/admin/validators.test.ts diff --git a/API.md b/API.md index ebea6ecd..a55631c6 100644 --- a/API.md +++ b/API.md @@ -300,6 +300,7 @@ These endpoints are all authenticated with admin authentication tokens, generate [documentation](https://permanent.atlassian.net/wiki/spaces/EN/pages/2072576001/Trigger+Admin+Directives) ### POST `/admin/folder/recalculate_thumbnails` +Queues thumbnail generation tasks for all non-root folders created between `beginTimestamp` and `endTimestamp`. - Headers: Authorization: Bearer \ @@ -307,7 +308,8 @@ These endpoints are all authenticated with admin authentication tokens, generate ``` { - cutoffTimestamp: string (date, iso format) + beginTimestamp: string (date, iso format) + endTimestamp: string (date, iso format) } ``` diff --git a/src/admin/controller.ts b/src/admin/controller.ts index ae8bed97..df7203db 100644 --- a/src/admin/controller.ts +++ b/src/admin/controller.ts @@ -13,7 +13,8 @@ adminController.post( try { validateRecalculateFolderThumbnailsRequest(req.body); const results = await adminService.recalculateFolderThumbnails( - req.body.cutoffTimestamp + req.body.beginTimestamp, + req.body.endTimestamp ); if (results.failedFolders.length > 0) { res.status(500).json(results); diff --git a/src/admin/queries/get_folders_created_before_timestamp.sql b/src/admin/queries/get_folders_created_before_timestamp.sql index 4e8d0b55..766eef15 100644 --- a/src/admin/queries/get_folders_created_before_timestamp.sql +++ b/src/admin/queries/get_folders_created_before_timestamp.sql @@ -4,4 +4,5 @@ SELECT FROM folder WHERE - createdDT < :cutoffTimestamp + createdDT BETWEEN :beginTimestamp AND :endTimestamp + AND (type IS NULL OR type NOT LIKE '%root%'); diff --git a/src/admin/service.test.ts b/src/admin/service.test.ts index 7647db1e..f9cb04f5 100644 --- a/src/admin/service.test.ts +++ b/src/admin/service.test.ts @@ -35,8 +35,11 @@ describe("recalculateFolderThumbnails", () => { test("should send messages for folders created before the cutoff", async () => { jest .spyOn(publisherClient, "batchPublishMessages") - .mockResolvedValueOnce({ failedMessages: [], messagesSent: 5 }); - const result = await adminService.recalculateFolderThumbnails(new Date()); + .mockResolvedValueOnce({ failedMessages: [], messagesSent: 4 }); + const result = await adminService.recalculateFolderThumbnails( + new Date(new Date().setDate(new Date().getDate() - 1)), + new Date() + ); expect( ( ( @@ -46,8 +49,8 @@ describe("recalculateFolderThumbnails", () => { ] )[1] as unknown as Message[] ).length - ).toBe(5); - expect(result).toEqual({ failedFolders: [], messagesSent: 5 }); + ).toBe(4); + expect(result).toEqual({ failedFolders: [], messagesSent: 4 }); }); test("should throw internal server error if database call fails", async () => { @@ -55,7 +58,10 @@ describe("recalculateFolderThumbnails", () => { const testError = new Error("out of cheese - redo from start"); jest.spyOn(db, "sql").mockRejectedValueOnce(testError); try { - await adminService.recalculateFolderThumbnails(new Date()); + await adminService.recalculateFolderThumbnails( + new Date(new Date().setDate(new Date().getDate() - 1)), + new Date() + ); } catch (err) { error = err; } finally { @@ -71,7 +77,10 @@ describe("recalculateFolderThumbnails", () => { .spyOn(publisherClient, "batchPublishMessages") .mockRejectedValueOnce(testError); try { - await adminService.recalculateFolderThumbnails(new Date()); + await adminService.recalculateFolderThumbnails( + new Date(new Date().setDate(new Date().getDate() - 1)), + new Date() + ); } catch (err) { error = err; } finally { diff --git a/src/admin/service.ts b/src/admin/service.ts index 897bd45a..8a349922 100644 --- a/src/admin/service.ts +++ b/src/admin/service.ts @@ -5,11 +5,13 @@ import { publisherClient, lowPriorityTopicArn } from "../publisher_client"; import { logger } from "../log"; const recalculateFolderThumbnails = async ( - cutoffTimestamp: Date + beginTimestamp: Date, + endTimestamp: Date ): Promise<{ messagesSent: number; failedFolders: string[] }> => { const folderResult = await db .sql("admin.queries.get_folders_created_before_timestamp", { - cutoffTimestamp, + beginTimestamp, + endTimestamp, }) .catch((err) => { logger.error(err); diff --git a/src/admin/validators.test.ts b/src/admin/validators.test.ts new file mode 100644 index 00000000..02295977 --- /dev/null +++ b/src/admin/validators.test.ts @@ -0,0 +1,150 @@ +import { validateRecalculateFolderThumbnailsRequest } from "./validators"; + +describe("validateRecalculateFolderThumbnailsRequest", () => { + test("should not error if the request is valid", () => { + let error = null; + try { + validateRecalculateFolderThumbnailsRequest({ + emailFromAuthToken: "test@permanent.org", + beginTimestamp: "2023-07-30", + endTimestamp: "2023-07-31", + }); + } catch (err) { + error = err; + } finally { + expect(error).toBeNull(); + } + }); + + test("should error if emailFromAuthToken is missing", () => { + let error = null; + try { + validateRecalculateFolderThumbnailsRequest({ + beginTimestamp: "2023-07-30", + endTimestamp: "2023-07-31", + }); + } catch (err) { + error = err; + } finally { + expect(error).not.toBeNull(); + } + }); + + test("should error if emailFromAuthToken is wrong type", () => { + let error = null; + try { + validateRecalculateFolderThumbnailsRequest({ + emailFromAuthToken: 123, + beginTimestamp: "2023-07-30", + endTimestamp: "2023-07-31", + }); + } catch (err) { + error = err; + } finally { + expect(error).not.toBeNull(); + } + }); + + test("should error if emailFromAuthToken is wrong format", () => { + let error = null; + try { + validateRecalculateFolderThumbnailsRequest({ + emailFromAuthToken: "not_an_email", + beginTimestamp: "2023-07-30", + endTimestamp: "2023-07-31", + }); + } catch (err) { + error = err; + } finally { + expect(error).not.toBeNull(); + } + }); + + test("should error if beginTimestamp is missing", () => { + let error = null; + try { + validateRecalculateFolderThumbnailsRequest({ + emailFromAuthToken: "test@permanent.org", + endTimestamp: "2023-07-31", + }); + } catch (err) { + error = err; + } finally { + expect(error).not.toBeNull(); + } + }); + + test("should error if beginTimestamp is wrong type", () => { + let error = null; + try { + validateRecalculateFolderThumbnailsRequest({ + emailFromAuthToken: "test@permanent.org", + beginTimestamp: "not_a_date", + endTimestamp: "2023-07-31", + }); + } catch (err) { + error = err; + } finally { + expect(error).not.toBeNull(); + } + }); + + test("should error if beginTimestamp is wrong format", () => { + let error = null; + try { + validateRecalculateFolderThumbnailsRequest({ + emailFromAuthToken: "test@permanent.org", + beginTimestamp: "07/31/23", + endTimestamp: "2023-07-31", + }); + } catch (err) { + error = err; + } finally { + expect(error).not.toBeNull(); + } + }); + + test("should error if endTimestamp is missing", () => { + let error = null; + try { + validateRecalculateFolderThumbnailsRequest({ + emailFromAuthToken: "test@permanent.org", + beginTimestamp: "2023-07-30", + }); + } catch (err) { + error = err; + } finally { + expect(error).not.toBeNull(); + } + }); + + test("should error if endTimestamp is wrong type", () => { + let error = null; + try { + validateRecalculateFolderThumbnailsRequest({ + emailFromAuthToken: "test@permanent.org", + beginTimestamp: "2023-07-30", + endTimestamp: "not_a_date", + }); + } catch (err) { + error = err; + } finally { + expect(error).not.toBeNull(); + } + }); + + test("should error if endTimestamp is wrong format", () => { + let error = null; + try { + validateRecalculateFolderThumbnailsRequest({ + emailFromAuthToken: "test@permanent.org", + beginTimestamp: "2023-07-30", + endTimestamp: "07/31/23", + }); + } catch (err) { + error = err; + } finally { + expect(error).not.toBeNull(); + } + }); +}); diff --git a/src/admin/validators.ts b/src/admin/validators.ts index 63fa6d21..e78a2c81 100644 --- a/src/admin/validators.ts +++ b/src/admin/validators.ts @@ -2,11 +2,12 @@ import Joi from "joi"; export function validateRecalculateFolderThumbnailsRequest( data: unknown -): asserts data is { cutoffTimestamp: Date } { +): asserts data is { beginTimestamp: Date; endTimestamp: Date } { const validation = Joi.object() .keys({ emailFromAuthToken: Joi.string().email().required(), - cutoffTimestamp: Joi.date().iso().required(), + beginTimestamp: Joi.date().iso().required(), + endTimestamp: Joi.date().iso().required(), }) .validate(data); if (validation.error) { diff --git a/src/fixtures/create_test_folders.sql b/src/fixtures/create_test_folders.sql index adf466dd..b6b267f3 100644 --- a/src/fixtures/create_test_folders.sql +++ b/src/fixtures/create_test_folders.sql @@ -1,7 +1,7 @@ INSERT INTO folder (folderId, archiveId, publicDt, displayName, status, createdDt) VALUES - (1, 1, '2023-06-21', 'Public Folder', 'status.generic.ok', CURRENT_TIMESTAMP), + (1, 1, '2023-06-21', 'Public Folder', 'status.generic.ok', CURRENT_TIMESTAMP - '2 days'::interval), (2, 1, NULL, 'Private Folder', 'status.generic.ok', CURRENT_TIMESTAMP), (3, 1, CURRENT_TIMESTAMP + '1 day'::interval, 'Future Public Folder', 'status.generic.ok', CURRENT_TIMESTAMP), (4, 1, '2023-06-21', 'Deleted Folder', 'status.generic.deleted', CURRENT_TIMESTAMP),