Apphosting emulator: gracefully handle secret access

src/apphosting/config.ts

@@ -1,4 +1,4 @@
-import { resolve, join, dirname } from "path";
+import { resolve, join, dirname, basename } from "path";
 import { writeFileSync } from "fs";
 import * as yaml from "yaml";
 
@@ -7,6 +7,11 @@
 import * as prompt from "../prompt";
 import * as dialogs from "./secrets/dialogs";
 import { AppHostingYamlConfig } from "./yaml";
+import { FirebaseError } from "../error";
+import { promptForAppHostingYaml } from "./utils";
+import { fetchSecrets } from "./secrets";
+import { logger } from "../logger";
+import { getOrPromptProject } from "../management/projects";
 
 export const APPHOSTING_BASE_YAML_FILE = "apphosting.yaml";
 export const APPHOSTING_LOCAL_YAML_FILE = "apphosting.local.yaml";
@@ -37,6 +42,9 @@
   env?: Env[];
 }
 
+const SECRET_CONFIG = "Secret";
+const EXPORTABLE_CONFIG = [SECRET_CONFIG];
+
 /**
  * Returns the absolute path for an app hosting backend root.
  *
@@ -65,8 +73,8 @@
 
 /**
  * Returns paths of apphosting config files in the given path
- * */
-export function listAppHostingFilesInPath(path: string) {
+ */
+export function listAppHostingFilesInPath(path: string): string[] {
   return fs
     .listFiles(path)
     .filter((file) => APPHOSTING_YAML_FILE_REGEX.test(file))
@@ -172,15 +180,79 @@
   dynamicDispatch.store(path, projectYaml);
 }
 
+/**
+ * Reads userGivenConfigFile and exports the secrets defined in that file by
+ * hitting Google Secret Manager. The secrets are written in plain text to an
+ * apphosting.local.yaml file as environment variables.
+ *
+ * If userGivenConfigFile is not given, user is prompted to select one of the
+ * discovered app hosting yaml files.
+ */
+export async function exportConfig(
+  cwd: string,
+  backendRoot: string,
+  projectId?: string,
+  userGivenConfigFile?: string,
+): Promise<void> {
+  const choices = await prompt.prompt({}, [
+    {
+      type: "checkbox",
+      name: "configurations",
+      message: "What configs would you like to export?",
+      choices: EXPORTABLE_CONFIG,
+    },
+  ]);
+
+  /**
+   * TODO: Update when supporting additional configurations. Currently only
+   * Secrets are exportable.
+   */
+  if (!choices.configurations.includes(SECRET_CONFIG)) {
+    logger.info("No configs selected to export");
+    return;
+  }
+
+  if (!projectId) {
+    const project = await getOrPromptProject({});
+    projectId = project.projectId;
+  }
+
+  let localAppHostingConfig: AppHostingYamlConfig = AppHostingYamlConfig.empty();
+
+  const localAppHostingConfigPath = resolve(backendRoot, APPHOSTING_LOCAL_YAML_FILE);
+  if (fs.fileExistsSync(localAppHostingConfigPath)) {
+    localAppHostingConfig = await AppHostingYamlConfig.loadFromFile(localAppHostingConfigPath);
+  }
+
+  const configToExport = await loadConfigToExportSecrets(cwd, userGivenConfigFile);
+  const secretsToExport = configToExport.secrets;
+  if (!secretsToExport) {
+    logger.info("No secrets found to export in the chosen App Hosting config files");
+    return;
+  }
+
+  const secretMaterial = await fetchSecrets(projectId, secretsToExport);
+  for (const [key, value] of secretMaterial) {
+    localAppHostingConfig.addEnvironmentVariable({
+      variable: key,
+      value: value,
+      availability: ["RUNTIME"],
+    });
+  }
+
+  // update apphosting.local.yaml
+  await localAppHostingConfig.upsertFile(localAppHostingConfigPath);
+  logger.info(`Wrote secrets as environment variables to ${APPHOSTING_LOCAL_YAML_FILE}.`);
+}
+
 /**
  * Given apphosting yaml config paths this function returns the
  * appropriate combined configuration.
  *
  * Environment specific config (i.e apphosting.<environment>.yaml) will
  * take precedence over the base config (apphosting.yaml).
- *
- * @param envYamlPath: Example: "/home/my-project/apphosting.staging.yaml"
- * @param baseYamlPath: Example: "/home/my-project/apphosting.yaml"
+ * @param envYamlPath Example: "/home/my-project/apphosting.staging.yaml"
+ * @param baseYamlPath Example: "/home/my-project/apphosting.yaml"
  */
 export async function loadConfigForEnvironment(
   envYamlPath: string,
@@ -199,3 +271,63 @@
 
   return envYamlConfig;
 }
+
+/**
+ * Returns the appropriate App Hosting YAML configuration for exporting secrets.
+ * @return The final merged config
+ */
+export async function loadConfigToExportSecrets(
+  cwd: string,
+  userGivenConfigFile?: string,
+): Promise<AppHostingYamlConfig> {
+  if (userGivenConfigFile && !APPHOSTING_YAML_FILE_REGEX.test(userGivenConfigFile)) {
+    throw new FirebaseError(
+      "Invalid apphosting yaml config file provided. File must be in format: 'apphosting.yaml' or 'apphosting.<environment>.yaml'",
+    );
+  }
+
+  const allConfigs = getValidConfigs(cwd);
+  let userGivenConfigFilePath: string;
+  if (userGivenConfigFile) {
+    if (!allConfigs.has(userGivenConfigFile)) {
+      throw new FirebaseError(
+        `The provided app hosting config file "${userGivenConfigFile}" does not exist`,
+      );
+    }
+
+    userGivenConfigFilePath = allConfigs.get(userGivenConfigFile)!;
+  } else {
+    userGivenConfigFilePath = await promptForAppHostingYaml(
+      allConfigs,
+      "Which environment would you like to export secrets from Secret Manager for?",
+    );
+  }
+
+  if (userGivenConfigFile === APPHOSTING_BASE_YAML_FILE) {
+    return AppHostingYamlConfig.loadFromFile(allConfigs.get(APPHOSTING_BASE_YAML_FILE)!);
+  }
+
+  const baseFilePath = allConfigs.get(APPHOSTING_BASE_YAML_FILE)!;
+  return await loadConfigForEnvironment(userGivenConfigFilePath, baseFilePath);
+}
+
+/**
+ * Gets all apphosting yaml configs excluding apphosting.local.yaml and returns
+ * a map in the format {"apphosting.staging.yaml" => "/cwd/apphosting.staging.yaml"}.
+ */
+function getValidConfigs(cwd: string): Map<string, string> {
+  const appHostingConfigPaths = listAppHostingFilesInPath(cwd).filter(
+    (path) => !path.endsWith(APPHOSTING_LOCAL_YAML_FILE),
+  );
+  if (appHostingConfigPaths.length === 0) {
+    throw new FirebaseError("No apphosting.*.yaml configs found");
+  }
+
+  const fileNameToPathMap: Map<string, string> = new Map();
+  for (const path of appHostingConfigPaths) {
+    const fileName = basename(path);
+    fileNameToPathMap.set(fileName, path);
+  }
+
+  return fileNameToPathMap;
+}

src/apphosting/secrets/index.spec.ts

@@ -9,10 +9,8 @@
 import * as gcsmImport from "../../gcp/secretManager";
 import * as utilsImport from "../../utils";
 import * as promptImport from "../../prompt";
-import * as configImport from "../config";
 
 import { Secret } from "../yaml";
-import { FirebaseError } from "../../error";
 
 describe("secrets", () => {
   let gcsm: sinon.SinonStubbedInstance<typeof gcsmImport>;
@@ -37,7 +35,7 @@
     it("uses explicit account", () => {
       const backend = {
         serviceAccount: "sa",
       } as any as apphosting.Backend;
       expect(secrets.serviceAccountsForBackend("number", backend)).to.deep.equal({
         buildServiceAccount: "sa",
         runServiceAccount: "sa",
@@ -45,7 +43,7 @@
     });
 
     it("has a fallback for legacy SAs", () => {
       const backend = {} as any as apphosting.Backend;
       expect(secrets.serviceAccountsForBackend("number", backend)).to.deep.equal({
         buildServiceAccount: gcb.getDefaultServiceAccount("number"),
         runServiceAccount: gce.getDefaultServiceAccount("number"),
@@ -297,47 +295,4 @@
       );
     });
   });
-
-  describe("loadConfigToExport", () => {
-    let apphostingConfigs: sinon.SinonStubbedInstance<typeof configImport>;
-
-    const baseYamlPath = "/parent/cwd/apphosting.yaml";
-
-    beforeEach(() => {
-      apphostingConfigs = sinon.stub(configImport);
-      apphostingConfigs.listAppHostingFilesInPath.returns([
-        baseYamlPath,
-        "/parent/apphosting.staging.yaml",
-      ]);
-    });
-
-    afterEach(() => {
-      sinon.verifyAndRestore();
-    });
-
-    it("returns throws an error if an invalid apphosting yaml if provided", async () => {
-      await expect(secrets.loadConfigToExport("/parent/cwd", "blah.txt")).to.be.rejectedWith(
-        FirebaseError,
-        "Invalid apphosting yaml config file provided. File must be in format: 'apphosting.yaml' or 'apphosting.<environment>.yaml'",
-      );
-    });
-
-    it("does not prompt user if an userGivenConfigFile is provided", async () => {
-      await secrets.loadConfigToExport("/parent/cwd", "apphosting.staging.yaml");
-      expect(prompt.promptOnce).to.not.be.called;
-    });
-
-    it("prompts user if userGivenConfigFile not provided", async () => {
-      await secrets.loadConfigToExport("/parent/cwd");
-      expect(prompt.promptOnce).to.be.called;
-    });
-
-    it("should throw an error if userGivenConfigFile could not be found", async () => {
-      await expect(
-        secrets.loadConfigToExport("/parent/cwd", "apphosting.preview.yaml"),
-      ).to.be.rejectedWith(
-        'The provided app hosting config file "apphosting.preview.yaml" does not exist',
-      );
-    });
-  });
 });

src/apphosting/secrets/index.ts

@@ -8,16 +8,7 @@
 import { isFunctionsManaged } from "../../gcp/secretManager";
 import * as utils from "../../utils";
 import * as prompt from "../../prompt";
-import { basename } from "path";
-import {
-  APPHOSTING_BASE_YAML_FILE,
-  APPHOSTING_LOCAL_YAML_FILE,
-  APPHOSTING_YAML_FILE_REGEX,
-  listAppHostingFilesInPath,
-  loadConfigForEnvironment,
-} from "../config";
-import { promptForAppHostingYaml } from "../utils";
-import { AppHostingYamlConfig, Secret } from "../yaml";
+import { Secret } from "../yaml";
 
 /** Interface for holding the service account pair for a given Backend. */
 export interface ServiceAccounts {
@@ -126,7 +117,7 @@
  * If a secret exists, we verify the user is not trying to change the region and verifies a secret
  * is not being used for both functions and app hosting as their garbage collection is incompatible
  * (client vs server-side).
  * @returns true if a secret was created, false if a secret already existed, and null if a user aborts.
  */
 export async function upsertSecret(
   project: string,
@@ -176,24 +167,43 @@
 }
 
 /**
- * Fetches secrets from Google Secret Manager and returns their values in plain text.
+ * Fetches secrets from Google Secret Manager and returns their values in plain text. If a secret is
+ * not accessible due to not having access permissions, it is not included in the returned map.
  */
 export async function fetchSecrets(
   projectId: string,
   secrets: Secret[],
 ): Promise<Map<string, string>> {
-  let secretsKeyValuePairs: Map<string, string>;
+  const secretsKeyValuePairs = new Map<string, string>();
+  const promises: Promise<void>[] = [];
 
   try {
-    const secretPromises: Promise<[string, string]>[] = secrets.map(async (secretConfig) => {
+    for (const secretConfig of secrets) {
       const [name, version] = getSecretNameParts(secretConfig.secret!);
 
-      const value = await gcsm.accessSecretVersion(projectId, name, version);
-      return [secretConfig.variable, value] as [string, string];
-    });
+      promises.push(
+        (async () => {
+          try {
+            const value = await gcsm.accessSecretVersion(projectId, name, version);
+            secretsKeyValuePairs.set(secretConfig.variable, value);
+          } catch (err: unknown) {
+            const status = getErrStatus(err);
+            if (status === 403) {
+              utils.logLabeledWarning(
+                "apphosting",
+                `You don't have permission to access secret: ${name}. Please ensure you have secretmanager.secretAccessor permission.`,
+              );
+              return;
+            }
+            throw new FirebaseError(`Failed to check access for secret: ${name}`, {
+              original: getError(err),
+            });
+          }
+        })(),
+      );
+    }
 
-    const secretEntries = await Promise.all(secretPromises);
-    secretsKeyValuePairs = new Map(secretEntries);
+    await Promise.all(promises);
   } catch (e: any) {
     throw new FirebaseError(`Error exporting secrets`, {
       original: e,
@@ -203,66 +213,6 @@
   return secretsKeyValuePairs;
 }
 
-/**
- * Returns the appropriate App Hosting YAML configuration for exporting secrets.
- *
- * @returns The final merged config
- */
-export async function loadConfigToExport(
-  cwd: string,
-  userGivenConfigFile?: string,
-): Promise<AppHostingYamlConfig> {
-  if (userGivenConfigFile && !APPHOSTING_YAML_FILE_REGEX.test(userGivenConfigFile)) {
-    throw new FirebaseError(
-      "Invalid apphosting yaml config file provided. File must be in format: 'apphosting.yaml' or 'apphosting.<environment>.yaml'",
-    );
-  }
-
-  const allConfigs = getValidConfigs(cwd);
-  let userGivenConfigFilePath: string;
-  if (userGivenConfigFile) {
-    if (!allConfigs.has(userGivenConfigFile)) {
-      throw new FirebaseError(
-        `The provided app hosting config file "${userGivenConfigFile}" does not exist`,
-      );
-    }
-
-    userGivenConfigFilePath = allConfigs.get(userGivenConfigFile)!;
-  } else {
-    userGivenConfigFilePath = await promptForAppHostingYaml(
-      allConfigs,
-      "Which environment would you like to export secrets from Secret Manager for?",
-    );
-  }
-
-  if (userGivenConfigFile === APPHOSTING_BASE_YAML_FILE) {
-    return AppHostingYamlConfig.loadFromFile(allConfigs.get(APPHOSTING_BASE_YAML_FILE)!);
-  }
-
-  const baseFilePath = allConfigs.get(APPHOSTING_BASE_YAML_FILE)!;
-  return await loadConfigForEnvironment(userGivenConfigFilePath, baseFilePath);
-}
-
-/**
- * Gets all apphosting yaml configs excluding apphosting.local.yaml and returns
- * a map in the format {"apphosting.staging.yaml" => "/cwd/apphosting.staging.yaml"}.
- */
-function getValidConfigs(cwd: string): Map<string, string> {
-  const appHostingConfigPaths = listAppHostingFilesInPath(cwd).filter(
-    (path) => !path.endsWith(APPHOSTING_LOCAL_YAML_FILE),
-  );
-  if (appHostingConfigPaths.length === 0) {
-    throw new FirebaseError("No apphosting.*.yaml configs found");
-  }
-
-  const fileNameToPathMap: Map<string, string> = new Map();
-  for (const path of appHostingConfigPaths) {
-    const fileName = basename(path);
-    fileNameToPathMap.set(fileName, path);
-  }
-
-  return fileNameToPathMap;
-}
 /**
  * secret expected to be in format "myApiKeySecret@5",
  * "projects/test-project/secrets/secretID", or