Prepare v1.4.0

CHANGELOG.md

@@ -7,6 +7,20 @@ This change log follows the format documented in [Keep a CHANGELOG].
 [semantic versioning]: https://semver.org
 [keep a changelog]: https://keepachangelog.com
 
+## 1.4.0 - 2022-06-03
+
+### Changed
+
+- Set default `memory` (`1GB`) and `timeoutSeconds` (`540`) Firebase Functions runtime options. It solves the problem with the huge users' backups that either run out of memory or timeout.
+
+- Even further improved the memory usage by the users backup.
+
+- Updated dependencies to the latest supported versions.
+
+### Added
+
+- Added delayed users backup feature. If the delay is requested, the agent will respond with a pending backup state. When the backup is completed, the agent will notify the controller. That prevents multiple backups caused by timeouts.
+
 ## 1.3.0 - 2022-06-01
 
 ### Fixed

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backupfire/firebase",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "Backup Fire Firebase agent",
   "keywords": [
     "backup Firebase database",

src/_lib/operation/index.ts

@@ -16,7 +16,10 @@ export type FirestoreStatusResponse =
 
 export type UsersStatusResponse =
   | {
-      state: 'completed' | 'pending'
+      state: 'pending'
+    }
+  | {
+      state: 'completed'
       data: {
         usersCount: number | undefined
         size: string

src/index.ts

@@ -8,39 +8,41 @@ import { format } from 'url'
 import {
   backupFirestoreMiddleware,
   checkFirestoreBackupStatusMiddleware,
-  getCollectionsMiddleware
+  getCollectionsMiddleware,
 } from './firestore'
 import {
-  listFilesMiddleware,
+  defaultControllerDomain,
+  defaultMemory,
+  defaultRegion,
+  defaultTimeout,
+} from './options'
+import {
   createStorageMiddleware,
+  listFilesMiddleware,
   storageListMiddleware,
-  updateStorageMiddleware
+  updateStorageMiddleware,
 } from './storage'
 import {
   AgentOptions,
   BackupFireEnvConfig,
   BackupFireHTTPSHandler,
   BackupFireOptions,
-  RuntimeEnvironment
+  RuntimeEnvironment,
 } from './types'
 import { backupUsersMiddleware } from './users'
 import version from './version'
 import {
   configureExceptionsScope,
   createCrashedApp,
   exceptionHandlerMiddleware,
-  initExceptionsTracker
+  initExceptionsTracker,
 } from './_lib/exceptions'
 
-export const defaultControllerDomain = 'backupfire.dev'
-
-export const defaultRegion = 'us-central1'
-
 export enum BackupFireConfig {
   Token = 'BACKUPFIRE_TOKEN',
   Password = 'BACKUPFIRE_PASSWORD',
   Domain = 'BACKUPFIRE_DOMAIN',
-  Allowlist = 'BACKUPFIRE_ALLOWLIST'
+  Allowlist = 'BACKUPFIRE_ALLOWLIST',
 }
 
 // Fallback for CommonJS
@@ -60,7 +62,7 @@ export default function backupFire(agentOptions?: AgentOptions) {
       return dummyHandler({
         region: agentOptions?.region,
         memory: agentOptions?.memory,
-        timeout: agentOptions?.timeout
+        timeout: agentOptions?.timeout,
       })
 
     // Derive Backup Fire options from environment configuration
@@ -80,7 +82,7 @@ export default function backupFire(agentOptions?: AgentOptions) {
         controllerToken: envConfig.token,
         adminPassword: envConfig.password,
         bucketsAllowlist: envConfig.allowlist?.split(','),
-        debug: envConfig.debug === 'true'
+        debug: envConfig.debug === 'true',
       },
       agentOptions
     )
@@ -111,7 +113,7 @@ export default function backupFire(agentOptions?: AgentOptions) {
     }
 
     // Set additional context
-    configureExceptionsScope(scope => {
+    configureExceptionsScope((scope) => {
       scope.setUser({ id: envConfig.token })
       scope.setTag('project_id', runtimeEnv.projectId)
       scope.setTag('node_version', process.version)
@@ -131,12 +133,12 @@ export default function backupFire(agentOptions?: AgentOptions) {
     return httpsHandler({
       handler: createApp(runtimeEnv, options),
       agentOptions,
-      runtimeEnv
+      runtimeEnv,
     })
   } catch (err) {
     return httpsHandler({
       handler: createCrashedApp(err),
-      agentOptions
+      agentOptions,
     })
   }
 }
@@ -173,7 +175,7 @@ export function createApp(
     '/firestore',
     backupFirestoreMiddleware({
       projectId: runtimeEnv.projectId,
-      ...globalOptions
+      ...globalOptions,
     })
   )
   // Check Firestore backup status
@@ -185,7 +187,13 @@ export function createApp(
   // Backup Firebase users
   app.post(
     '/users',
-    backupUsersMiddleware({ projectId: runtimeEnv.projectId, ...globalOptions })
+    backupUsersMiddleware({
+      projectId: runtimeEnv.projectId,
+      controllerToken: options.controllerToken,
+      controllerDomain: options.controllerDomain,
+      agentURL: agentURL(runtimeEnv),
+      ...globalOptions,
+    })
   )
 
   // List storage
@@ -197,7 +205,7 @@ export function createApp(
     '/storage/:storageId',
     updateStorageMiddleware({
       adminPassword: options.adminPassword,
-      ...globalOptions
+      ...globalOptions,
     })
   )
   // List files in the storage
@@ -217,21 +225,16 @@ interface HTTPSHandlerProps {
 function httpsHandler({
   handler,
   agentOptions,
-  runtimeEnv
+  runtimeEnv,
 }: HTTPSHandlerProps) {
   if (runtimeEnv?.extensionId) {
     return functions.handler.https.onRequest(handler)
   } else {
-    const runtimeOptions: functions.RuntimeOptions = {
-      secrets: Object.values(BackupFireConfig)
-    }
-
-    if (agentOptions?.memory) runtimeOptions.memory = agentOptions.memory
-    if (agentOptions?.timeout)
-      runtimeOptions.timeoutSeconds = agentOptions.timeout
-
     return functions
-      .runWith(runtimeOptions)
+      .runWith({
+        ...getRuntimeOptions(agentOptions),
+        secrets: Object.values(BackupFireConfig),
+      })
       .region(agentOptions?.region || defaultRegion)
       .https.onRequest(handler)
   }
@@ -253,8 +256,8 @@ function sendInitializationPing(
       token: options.controllerToken,
       projectId: runtimeEnv.projectId,
       runtime: runtimeEnv.region,
-      agentURL: agentURL(runtimeEnv)
-    }
+      agentURL: agentURL(runtimeEnv),
+    },
   })
   return fetch(pingURL)
 }
@@ -290,7 +293,7 @@ function getRuntimeEnv(
     // Node.js v8 runtime uses FUNCTION_NAME, v10 — FUNCTION_TARGET
     // See: https://cloud.google.com/functions/docs/env-var#environment_variables_set_automatically
     functionName: process.env.FUNCTION_NAME || process.env.FUNCTION_TARGET,
-    extensionId
+    extensionId,
   }
 }
 
@@ -326,7 +329,7 @@ function dummyHandler(
   if (options?.timeout) runtimeOptions.timeoutSeconds = options.timeout
 
   return functions
-    .runWith(runtimeOptions)
+    .runWith(getRuntimeOptions(options))
     .region(options.region || defaultRegion)
     .https.onRequest((_req, resp) => {
       resp.end()
@@ -336,3 +339,24 @@ function dummyHandler(
 function prettyJSON(obj: any) {
   return JSON.stringify(obj, null, 2)
 }
+
+/**
+ *
+ * @param agentOptions - TODO
+ * @returns
+ */
+function getRuntimeOptions(
+  agentOptions: AgentOptions | undefined
+): functions.RuntimeOptions {
+  const options: functions.RuntimeOptions = {
+    // Always assign timeout to runtime options. Unless the user defines
+    // a custom timeout, we want to use the default timeout of 9 minutes,
+    // to make sure the user backups are completed regardless of how many
+    // there are.
+    timeoutSeconds: agentOptions?.timeout || defaultTimeout,
+  }
+
+  if (agentOptions?.memory) options.memory = agentOptions.memory
+
+  return options
+}

src/options.ts

@@ -0,0 +1,21 @@
+export const defaultControllerDomain = 'backupfire.dev'
+
+export const defaultRegion = 'us-central1'
+
+/**
+ * The default function timeout - 9 minutes. It ensures that the user backups
+ * are completed regardless of how many there are.
+ *
+ * Unlike the memory runtime option, timeout doesn't affect the function
+ * instance price, so it's safe to set it to max.
+ */
+export const defaultTimeout = 540
+
+/**
+ * The default function memory. With the increased timeout, it ensures
+ * the users' backup completion.
+ *
+ * Internal testing shows that 1GB is the sweet spot. It's still cheap to run
+ * and gives room to process huge backups.
+ */
+export const defaultMemory = '1GB'