Merge pull request #400 from lepikhinb/chunk-keys-to-delete

Chunk deleted files into batches
getlift · Aug 31, 2024 · 7cdbb93 · 7cdbb93
2 parents 8c2716c + 4854e6c
commit 7cdbb93
Showing 1 changed file with 16 additions and 5 deletions.
diff --git a/src/utils/s3-sync.ts b/src/utils/s3-sync.ts
@@ -76,13 +76,24 @@ export async function s3Sync({
     const targetKeys = filesToUpload.map((file) =>
         targetPathPrefix !== undefined ? path.posix.join(targetPathPrefix, file) : file
     );
+
     const keysToDelete = findKeysToDelete(Object.keys(existingS3Objects), targetKeys);
     if (keysToDelete.length > 0) {
-        keysToDelete.map((key) => {
-            getUtils().log.verbose(`Deleting ${key}`);
-            fileChangeCount++;
-        });
-        await s3Delete(aws, bucketName, keysToDelete);
+        const batch = chunk(keysToDelete, 1000);
+
+        await Promise.all(
+            batch.map(async (keysToDeleteChunk, index) => {
+                getUtils().log.verbose(`Deleting chunk #${index}`);
+
+                keysToDeleteChunk.map((key) => {
+                    getUtils().log.verbose(`Deleting ${key}`);
+                    fileChangeCount++;
+                });
+
+                await s3Delete(aws, bucketName, keysToDeleteChunk);
+            })
+        );
+
         hasChanges = true;
     }