feat(metrics): add Glean for backend metrics

Because:
 - we want to use Glean for backend metrics

This commit:
 - adds the yamls and glean parser generated TS code
 - adds Glean related config for auth-server
 - adds a lib to gather metric values and calling the generated code
 - logs the "login_success" event when a user successfully get a
   verified session on the account login route

packages/fxa-auth-server/bin/key_server.js

@@ -29,6 +29,7 @@ const {
 const { setupFirestore } = require('../lib/firestore-db');
 const { AppleIAP } = require('../lib/payments/iap/apple-app-store/apple-iap');
 const { AccountEventsManager } = require('../lib/account-events');
+const { gleanMetrics } = require('../lib/metrics/glean');
 
 async function run(config) {
   Container.set(AppConfig, config);
@@ -158,6 +159,8 @@ async function run(config) {
   const zendeskClient = require('../lib/zendesk-client').createZendeskClient(
     config
   );
+  const glean = gleanMetrics(config);
+
   const routes = require('../lib/routes')(
     log,
     serverPublicKeys,
@@ -171,7 +174,8 @@ async function run(config) {
     statsd,
     profile,
     stripeHelper,
-    redis
+    redis,
+    glean
   );
 
   const Server = require('../lib/server');

packages/fxa-auth-server/config/index.ts

@@ -1955,6 +1955,32 @@ const convictConf = convict({
       format: Boolean,
     },
   },
+  gleanMetrics: {
+    enabled: {
+      default: true,
+      doc: 'Enable Glean metrics logging',
+      env: 'AUTH_GLEAN_ENABLED',
+      format: Boolean,
+    },
+    applicationId: {
+      default: 'accounts_backend_dev',
+      doc: 'The Glean application id',
+      env: 'AUTH_GLEAN_APP_ID',
+      format: String,
+    },
+    channel: {
+      default: 'development',
+      doc: 'The application channel, e.g. development, stage, production, etc.',
+      env: 'AUTH_GLEAN_APP_CHANNEL',
+      format: String,
+    },
+    loggerAppName: {
+      default: 'fxa-auth-api',
+      doc: 'Used to form the mozlog logger name',
+      env: 'AUTH_GLEAN_LOGGER_APP_NAME',
+      format: String,
+    },
+  },
 });
 
 // handle configuration files.  you can specify a CSV list of configuration

packages/fxa-auth-server/lib/metrics/glean/index.ts

@@ -0,0 +1,118 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { ConfigType } from '../../../config';
+import { createAccountsEventsEvent } from './server_events';
+import { version } from '../../../package.json';
+import { createHash } from 'crypto';
+import { AuthRequest } from '../../types';
+
+// According to @types/hapi, request.auth.credentials.user is of type
+// UserCredentials, which is just {}. That's not actually the case and it
+// mismatches the real type, which is string.  I extended AuthRequest below but
+// the type, MetricsRquest is for this file only.
+interface MetricsRequest extends Omit<AuthRequest, 'auth'> {
+  payload: Record<string, any>;
+  auth: { credentials: Record<string, string> };
+}
+
+type MetricsData = {
+  uid?: string;
+};
+
+let appConfig: ConfigType;
+let gleanEventLogger: ReturnType<typeof createAccountsEventsEvent>;
+
+const isEnabled = async (request: MetricsRequest) =>
+  appConfig.gleanMetrics.enabled && (await request.app.isMetricsEnabled);
+
+const findUid = (request: MetricsRequest, metricsData?: MetricsData): string =>
+  metricsData?.uid ||
+  request.auth?.credentials?.uid ||
+  request.auth?.credentials?.user ||
+  '';
+
+const sha256HashUid = (uid: string) =>
+  createHash('sha256').update(uid).digest('hex');
+
+const findOauthClientId = (request: MetricsRequest): string =>
+  request.auth.credentials?.client_id || request.payload.client_id || '';
+
+const findServiceName = async (request: MetricsRequest) => {
+  const metricsContext = await request.app.metricsContext;
+
+  if (metricsContext.service) {
+    return metricsContext.service;
+  }
+
+  const clientId = findOauthClientId(request);
+
+  // use the client id to service name mapping from the app config
+  if (clientId && appConfig.oauth.clientIds[clientId]) {
+    return appConfig.oauth.clientIds[clientId];
+  }
+
+  return '';
+};
+
+const createEventFn =
+  // On MetricsData: for an event like successful login, the uid isn't known at
+  // the time of request since the request itself isn't authenticated.  We'll
+  // accept data from the event logging call for metrics that are known/easily
+  // accessible in the calling scope but difficult/not possible to get from any
+  // context attached to the request.
+
+
+    (eventName: string) =>
+    async (req: AuthRequest, metricsData?: MetricsData) => {
+      // where the function is called the request object is likely to be declared
+      // to be AuthRequest, so we do a cast here.
+      const request = req as unknown as MetricsRequest;
+      const enabled = await isEnabled(request);
+      if (!enabled) {
+        return;
+      }
+
+      const metricsContext = await request.app.metricsContext;
+
+      const metrics = {
+        account_user_id_sha256: '',
+        event_name: eventName,
+        event_reason: '',
+        relying_party_oauth_client_id: findOauthClientId(request),
+        relying_party_service: await findServiceName(request),
+        session_device_type: request.app.ua.deviceType || '',
+        session_entrypoint: metricsContext.entrypoint || '',
+        session_flow_id: metricsContext.flowId || '',
+        utm_campaign: metricsContext.utmCampaign || '',
+        utm_content: metricsContext.utmContent || '',
+        utm_medium: metricsContext.utmMedium || '',
+        utm_source: metricsContext.utmSource || '',
+        utm_term: metricsContext.utmTerm || '',
+      };
+
+      // uid needs extra handling because we need to hash the value
+      const uid = findUid(request, metricsData);
+      if (uid !== '') {
+        metrics.account_user_id_sha256 = sha256HashUid(uid);
+      }
+
+      await gleanEventLogger.record(metrics);
+    };
+
+export const gleanMetrics = (config: ConfigType) => {
+  appConfig = config;
+  gleanEventLogger = createAccountsEventsEvent({
+    applicationId: config.gleanMetrics.applicationId,
+    appDisplayVersion: version,
+    channel: config.gleanMetrics.channel,
+    logger_options: { app: config.gleanMetrics.loggerAppName },
+  });
+
+  return {
+    login: {
+      success: createEventFn('login_success'),
+    },
+  };
+};

packages/fxa-auth-server/lib/metrics/glean/server_events.ts

@@ -0,0 +1,165 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// AUTOGENERATED BY glean_parser v8.1.1. DO NOT EDIT. DO NOT COMMIT.
+
+// This requires `uuid` and `mozlog` libraries to be in the environment
+// @types/uuid and mozlog types definitions are required in devDependencies
+// for the latter see https://github.com/mozilla/fxa/blob/85bda71cda376c417b8c850ba82aa14252208c3c/types/mozlog/index.d.ts
+import { v4 as uuidv4 } from 'uuid';
+import mozlog, { Logger } from 'mozlog';
+
+const GLEAN_EVENT_MOZLOG_TYPE = 'glean-server-event';
+type LoggerOptions = { app: string; fmt?: 'heka' };
+
+let _logger: Logger;
+
+class AccountsEventsServerEvent {
+  _applicationId: string;
+  _appDisplayVersion: string;
+  _channel: string;
+  /**
+   * Create AccountsEventsServerEvent instance.
+   *
+   * @param {string} applicationId - The application ID.
+   * @param {string} appDisplayVersion - The application display version.
+   * @param {string} channel - The channel.
+   */
+  constructor(
+    applicationId: string,
+    appDisplayVersion: string,
+    channel: string,
+    logger_options: LoggerOptions
+  ) {
+    this._applicationId = applicationId;
+    this._appDisplayVersion = appDisplayVersion;
+    this._channel = channel;
+
+    if (!_logger) {
+      // append '-glean' to `logger_options.app` to avoid collision with other loggers and double logging
+      logger_options.app = logger_options.app + '-glean';
+      // set the format to `heka` so messages are properly ingested and decoded
+      logger_options.fmt = 'heka';
+      // mozlog types declaration requires a typePrefix to be passed when creating a logger
+      // we don't want a typePrefix, so we pass `undefined`
+      _logger = mozlog(logger_options)(undefined);
+    }
+  }
+  /**
+   * Record and submit a server event object.
+   * Event is logged using internal mozlog logger.
+   *
+   * @param {string} account_user_id_sha256 - A hex string of a sha256 hash of the account's uid.
+   * @param {string} event_name - The name of the event.
+   * @param {string} event_reason - additional context-dependent (on event.name) info, e.g. the cause of an error.
+   * @param {string} relying_party_oauth_client_id - The client id of the relying party.
+   * @param {string} relying_party_service - The service name of the relying party.
+   * @param {string} session_device_type - one of 'mobile', 'tablet', or ''.
+   * @param {string} session_entrypoint - entrypoint to the service.
+   * @param {string} session_flow_id - an ID generated by FxA for its flow metrics.
+   * @param {string} utm_campaign - A marketing campaign.  For example, if a user signs into FxA from selecting a Mozilla VPN plan on Mozilla VPN's product site, then value of this metric could be 'vpn-product-page'.  The value has a max length of 128 characters with the alphanumeric characters, _ (underscore), forward slash (/), . (period), % (percentage sign), and - (hyphen) in the allowed set of characters.  The special value of 'page+referral+-+not+part+of+a+campaign' is also allowed..
+   * @param {string} utm_content - The content on which the user acted.  For example, if the user clicked on the "Get started here" link in "Looking for Firefox Sync? Get started here", then the value for this metric would be 'fx-sync-get-started'.  The value has a max length of 128 characters with the alphanumeric characters, _ (underscore), forward slash (/), . (period), % (percentage sign), and - (hyphen) in the allowed set of characters..
+   * @param {string} utm_medium - The "medium" on which the user acted.  For example, if the user clicked on a link in an email, then the value of this metric would be 'email'.  The value has a max length of 128 characters with the alphanumeric characters, _ (underscore), forward slash (/), . (period), % (percentage sign), and - (hyphen) in the allowed set of characters..
+   * @param {string} utm_source - The source from where the user started.  For example, if the user clicked on a link on the Firefox accounts web site, this value could be 'fx-website'.  The value has a max length of 128 characters with the alphanumeric characters, _ (underscore), forward slash (/), . (period), % (percentage sign), and - (hyphen) in the allowed set of characters..
+   * @param {string} utm_term - This metric is similar to the `utm.source`; it is used in the Firefox browser.  For example, if the user started from about:welcome, then the value could be 'aboutwelcome-default-screen'.  The value has a max length of 128 characters with the alphanumeric characters, _ (underscore), forward slash (/), . (period), % (percentage sign), and - (hyphen) in the allowed set of characters..
+   */
+  record({
+    account_user_id_sha256,
+    event_name,
+    event_reason,
+    relying_party_oauth_client_id,
+    relying_party_service,
+    session_device_type,
+    session_entrypoint,
+    session_flow_id,
+    utm_campaign,
+    utm_content,
+    utm_medium,
+    utm_source,
+    utm_term,
+  }: {
+    account_user_id_sha256: string;
+    event_name: string;
+    event_reason: string;
+    relying_party_oauth_client_id: string;
+    relying_party_service: string;
+    session_device_type: string;
+    session_entrypoint: string;
+    session_flow_id: string;
+    utm_campaign: string;
+    utm_content: string;
+    utm_medium: string;
+    utm_source: string;
+    utm_term: string;
+  }) {
+    const timestamp = new Date().toISOString();
+    const eventPayload = {
+      metrics: {
+        string: {
+          'account.user_id_sha256': account_user_id_sha256,
+          'event.name': event_name,
+          'event.reason': event_reason,
+          'relying_party.oauth_client_id': relying_party_oauth_client_id,
+          'relying_party.service': relying_party_service,
+          'session.device_type': session_device_type,
+          'session.entrypoint': session_entrypoint,
+          'session.flow_id': session_flow_id,
+          'utm.campaign': utm_campaign,
+          'utm.content': utm_content,
+          'utm.medium': utm_medium,
+          'utm.source': utm_source,
+          'utm.term': utm_term,
+        },
+      },
+      ping_info: {
+        seq: 0, // this is required, however doesn't seem to be useful in server context
+        start_time: timestamp,
+        end_time: timestamp,
+      },
+      // `Unknown` fields below are required in the Glean schema, however they are not useful in server context
+      client_info: {
+        telemetry_sdk_build: 'glean_parser v8.1.1',
+        first_run_date: 'Unknown',
+        os: 'Unknown',
+        os_version: 'Unknown',
+        architecture: 'Unknown',
+        app_build: 'Unknown',
+        app_display_version: this._appDisplayVersion,
+        app_channel: this._channel,
+      },
+    };
+    const eventPayloadSerialized = JSON.stringify(eventPayload);
+
+    // This is the message structure that Decoder expects: https://github.com/mozilla/gcp-ingestion/pull/2400
+    const ping = {
+      document_namespace: this._applicationId,
+      document_type: 'accounts-events',
+      document_version: '1',
+      document_id: uuidv4(),
+      payload: eventPayloadSerialized,
+    };
+
+    // this is similar to how FxA currently logs with mozlog: https://github.com/mozilla/fxa/blob/4c5c702a7fcbf6f8c6b1f175e9172cdd21471eac/packages/fxa-auth-server/lib/log.js#L289
+    _logger.info(GLEAN_EVENT_MOZLOG_TYPE, ping);
+  }
+}
+
+export const createAccountsEventsEvent = function ({
+  applicationId,
+  appDisplayVersion,
+  channel,
+  logger_options,
+}: {
+  applicationId: string;
+  appDisplayVersion: string;
+  channel: string;
+  logger_options: LoggerOptions;
+}) {
+  return new AccountsEventsServerEvent(
+    applicationId,
+    appDisplayVersion,
+    channel,
+    logger_options
+  );
+};

packages/fxa-auth-server/lib/routes/account.ts

@@ -45,6 +45,7 @@ import emailUtils from './utils/email';
 import requestHelper from './utils/request_helper';
 import validators from './validators';
 import { AccountEventsManager } from '../account-events';
+import { gleanMetrics } from '../metrics/glean';
 
 const METRICS_CONTEXT_SCHEMA = require('../metrics/context').schema;
 
@@ -79,7 +80,8 @@ export class AccountHandler {
     private subscriptionAccountReminders: any,
     private oauth: any,
     private stripeHelper: StripeHelper,
-    private pushbox: any
+    private pushbox: any,
+    private glean: ReturnType<typeof gleanMetrics>
   ) {
     this.otpUtils = require('./utils/otp')(log, config, db);
     this.skipConfirmationForEmailAddresses = config.signinConfirmation
@@ -1112,6 +1114,10 @@ export class AccountHandler {
 
       await this.signinUtils.cleanupReminders(response, accountRecord);
 
+      if (response.verified) {
+        this.glean.login.success(request, { uid: sessionToken.uid });
+      }
+
       return response;
     };
 
@@ -1681,7 +1687,8 @@ export const accountRoutes = (
   subscriptionAccountReminders: any,
   oauth: any,
   stripeHelper: StripeHelper,
-  pushbox: any
+  pushbox: any,
+  glean: ReturnType<typeof gleanMetrics>
 ) => {
   const accountHandler = new AccountHandler(
     log,
@@ -1697,7 +1704,8 @@ export const accountRoutes = (
     subscriptionAccountReminders,
     oauth,
     stripeHelper,
-    pushbox
+    pushbox,
+    glean
   );
   const routes = [
     {