Fix generation of dynamic mapping for object with specific subfield (e…

…lastic#204104)

Fix generation of dynamic mapping for objects that have more specific
subfields in separate definitions.

This can be reproduced for example with:
```
- name: labels
  type: object
  object_type: keyword
  object_type_mapping_type: '*'
- name: labels.count
  type: long
```

Fleet expands and deduplicates field definitions before generating the
mappings, so the definitions above are converted to something like the
following:
```
- name: labels
  type: group
  object_type: keyword
  object_type_mapping_type: '*'
  fields:
  - name: count
    type: long
```

Usually fields of type `group` don't have an `object_type`, so this was
being ignored, the dynamic mapping was not being generated.

This issue was not reproduced if the object field name includes a
wildcard, like in `labels.*`, because then the expansion and
deduplication resolves to something like this:
```
- name: labels
  type: group
  fields:
  - name: '*'
    type: object
    object_type: keyword
    object_type_mapping_type: '*'
  - name: count
    type: long
```

(cherry picked from commit e3877e0)

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts

@@ -786,6 +786,84 @@ describe('EPM template', () => {
     expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
   });
 
+  it('tests processing object field with more specific properties without wildcard', () => {
+    const objectFieldWithPropertyReversedLiteralYml = `
+- name: labels
+  type: object
+  object_type: keyword
+  object_type_mapping_type: '*'
+- name: labels.count
+  type: long
+`;
+    const objectFieldWithPropertyReversedMapping = {
+      dynamic_templates: [
+        {
+          labels: {
+            path_match: 'labels.*',
+            match_mapping_type: '*',
+            mapping: {
+              type: 'keyword',
+            },
+          },
+        },
+      ],
+      properties: {
+        labels: {
+          dynamic: true,
+          type: 'object',
+          properties: {
+            count: {
+              type: 'long',
+            },
+          },
+        },
+      },
+    };
+    const fields: Field[] = load(objectFieldWithPropertyReversedLiteralYml);
+    const processedFields = processFields(fields);
+    const mappings = generateMappings(processedFields);
+    expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
+  });
+
+  it('tests processing object field with more specific properties with wildcard', () => {
+    const objectFieldWithPropertyReversedLiteralYml = `
+- name: labels.*
+  type: object
+  object_type: keyword
+  object_type_mapping_type: '*'
+- name: labels.count
+  type: long
+`;
+    const objectFieldWithPropertyReversedMapping = {
+      dynamic_templates: [
+        {
+          'labels.*': {
+            path_match: 'labels.*',
+            match_mapping_type: '*',
+            mapping: {
+              type: 'keyword',
+            },
+          },
+        },
+      ],
+      properties: {
+        labels: {
+          dynamic: true,
+          type: 'object',
+          properties: {
+            count: {
+              type: 'long',
+            },
+          },
+        },
+      },
+    };
+    const fields: Field[] = load(objectFieldWithPropertyReversedLiteralYml);
+    const processedFields = processFields(fields);
+    const mappings = generateMappings(processedFields);
+    expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
+  });
+
   it('tests processing object field with subobjects set to false (case B)', () => {
     const objectFieldWithPropertyReversedLiteralYml = `
 - name: b.labels.*

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts

@@ -299,6 +299,124 @@ function _generateMappings(
     }
   }
 
+  function addObjectAsDynamicMapping(field: Field) {
+    const path = ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name;
+    const pathMatch = path.includes('*') ? path : `${path}.*`;
+
+    let dynProperties: Properties = getDefaultProperties(field);
+    let matchingType: string | undefined;
+    switch (field.object_type) {
+      case 'histogram':
+        dynProperties = histogram(field);
+        matchingType = field.object_type_mapping_type ?? '*';
+        break;
+      case 'ip':
+      case 'keyword':
+      case 'match_only_text':
+      case 'text':
+      case 'wildcard':
+        dynProperties.type = field.object_type;
+        matchingType = field.object_type_mapping_type ?? 'string';
+        break;
+      case 'scaled_float':
+        dynProperties = scaledFloat(field);
+        matchingType = field.object_type_mapping_type ?? '*';
+        break;
+      case 'aggregate_metric_double':
+        dynProperties.type = field.object_type;
+        dynProperties.metrics = field.metrics;
+        dynProperties.default_metric = field.default_metric;
+        matchingType = field.object_type_mapping_type ?? '*';
+        break;
+      case 'double':
+      case 'float':
+      case 'half_float':
+        dynProperties.type = field.object_type;
+        if (isIndexModeTimeSeries) {
+          dynProperties.time_series_metric = field.metric_type;
+        }
+        matchingType = field.object_type_mapping_type ?? 'double';
+        break;
+      case 'byte':
+      case 'long':
+      case 'short':
+      case 'unsigned_long':
+        dynProperties.type = field.object_type;
+        if (isIndexModeTimeSeries) {
+          dynProperties.time_series_metric = field.metric_type;
+        }
+        matchingType = field.object_type_mapping_type ?? 'long';
+        break;
+      case 'integer':
+        // Map integers as long, as in other cases.
+        dynProperties.type = 'long';
+        if (isIndexModeTimeSeries) {
+          dynProperties.time_series_metric = field.metric_type;
+        }
+        matchingType = field.object_type_mapping_type ?? 'long';
+        break;
+      case 'boolean':
+        dynProperties.type = field.object_type;
+        if (isIndexModeTimeSeries) {
+          dynProperties.time_series_metric = field.metric_type;
+        }
+        matchingType = field.object_type_mapping_type ?? field.object_type;
+        break;
+      case 'group':
+        if (!field?.fields) {
+          break;
+        }
+        const subFields = field.fields.map((subField) => ({
+          ...subField,
+          type: 'object',
+          object_type: subField.object_type ?? subField.type,
+        }));
+        const mappings = _generateMappings(
+          subFields,
+          {
+            ...ctx,
+            groupFieldName: ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name,
+          },
+          isIndexModeTimeSeries
+        );
+        if (mappings.hasDynamicTemplateMappings) {
+          hasDynamicTemplateMappings = true;
+        }
+        break;
+      case 'flattened':
+        dynProperties.type = field.object_type;
+        matchingType = field.object_type_mapping_type ?? 'object';
+        break;
+      default:
+        throw new PackageInvalidArchiveError(
+          `No dynamic mapping generated for field ${path} of type ${field.object_type}`
+        );
+    }
+
+    if (field.dimension && isIndexModeTimeSeries) {
+      dynProperties.time_series_dimension = field.dimension;
+    }
+
+    // When a wildcard field specifies the subobjects setting,
+    // the parent intermediate object should set the subobjects
+    // setting.
+    //
+    // For example, if a wildcard field `foo.*` has subobjects,
+    // we should set subobjects on the intermediate object `foo`.
+    //
+    if (field.subobjects !== undefined && path.includes('*')) {
+      subobjects = field.subobjects;
+    }
+
+    if (dynProperties && matchingType) {
+      addDynamicMappingWithIntermediateObjects(path, pathMatch, matchingType, dynProperties);
+
+      // Add the parent object as static property, this is needed for
+      // index templates not using `"dynamic": true`.
+      addParentObjectAsStaticProperty(field);
+    }
+  }
+
   // TODO: this can happen when the fields property in fields.yml is present but empty
   // Maybe validation should be moved to fields/field.ts
   if (fields) {
@@ -360,123 +478,7 @@ function _generateMappings(
       }
 
       if (type === 'object' && field.object_type) {
-        const path = ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name;
-        const pathMatch = path.includes('*') ? path : `${path}.*`;
-
-        let dynProperties: Properties = getDefaultProperties(field);
-        let matchingType: string | undefined;
-        switch (field.object_type) {
-          case 'histogram':
-            dynProperties = histogram(field);
-            matchingType = field.object_type_mapping_type ?? '*';
-            break;
-          case 'ip':
-          case 'keyword':
-          case 'match_only_text':
-          case 'text':
-          case 'wildcard':
-            dynProperties.type = field.object_type;
-            matchingType = field.object_type_mapping_type ?? 'string';
-            break;
-          case 'scaled_float':
-            dynProperties = scaledFloat(field);
-            matchingType = field.object_type_mapping_type ?? '*';
-            break;
-          case 'aggregate_metric_double':
-            dynProperties.type = field.object_type;
-            dynProperties.metrics = field.metrics;
-            dynProperties.default_metric = field.default_metric;
-            matchingType = field.object_type_mapping_type ?? '*';
-            break;
-          case 'double':
-          case 'float':
-          case 'half_float':
-            dynProperties.type = field.object_type;
-            if (isIndexModeTimeSeries) {
-              dynProperties.time_series_metric = field.metric_type;
-            }
-            matchingType = field.object_type_mapping_type ?? 'double';
-            break;
-          case 'byte':
-          case 'long':
-          case 'short':
-          case 'unsigned_long':
-            dynProperties.type = field.object_type;
-            if (isIndexModeTimeSeries) {
-              dynProperties.time_series_metric = field.metric_type;
-            }
-            matchingType = field.object_type_mapping_type ?? 'long';
-            break;
-          case 'integer':
-            // Map integers as long, as in other cases.
-            dynProperties.type = 'long';
-            if (isIndexModeTimeSeries) {
-              dynProperties.time_series_metric = field.metric_type;
-            }
-            matchingType = field.object_type_mapping_type ?? 'long';
-            break;
-          case 'boolean':
-            dynProperties.type = field.object_type;
-            if (isIndexModeTimeSeries) {
-              dynProperties.time_series_metric = field.metric_type;
-            }
-            matchingType = field.object_type_mapping_type ?? field.object_type;
-            break;
-          case 'group':
-            if (!field?.fields) {
-              break;
-            }
-            const subFields = field.fields.map((subField) => ({
-              ...subField,
-              type: 'object',
-              object_type: subField.object_type ?? subField.type,
-            }));
-            const mappings = _generateMappings(
-              subFields,
-              {
-                ...ctx,
-                groupFieldName: ctx.groupFieldName
-                  ? `${ctx.groupFieldName}.${field.name}`
-                  : field.name,
-              },
-              isIndexModeTimeSeries
-            );
-            if (mappings.hasDynamicTemplateMappings) {
-              hasDynamicTemplateMappings = true;
-            }
-            break;
-          case 'flattened':
-            dynProperties.type = field.object_type;
-            matchingType = field.object_type_mapping_type ?? 'object';
-            break;
-          default:
-            throw new PackageInvalidArchiveError(
-              `No dynamic mapping generated for field ${path} of type ${field.object_type}`
-            );
-        }
-
-        if (field.dimension && isIndexModeTimeSeries) {
-          dynProperties.time_series_dimension = field.dimension;
-        }
-
-        // When a wildcard field specifies the subobjects setting,
-        // the parent intermediate object should set the subobjects
-        // setting.
-        //
-        // For example, if a wildcard field `foo.*` has subobjects,
-        // we should set subobjects on the intermediate object `foo`.
-        //
-        if (field.subobjects !== undefined && path.includes('*')) {
-          subobjects = field.subobjects;
-        }
-
-        if (dynProperties && matchingType) {
-          addDynamicMappingWithIntermediateObjects(path, pathMatch, matchingType, dynProperties);
-
-          // Add the parent object as static property, this is needed for
-          // index templates not using `"dynamic": true`.
-          addParentObjectAsStaticProperty(field);
-        }
+        addObjectAsDynamicMapping(field);
       } else {
         let fieldProps = getDefaultProperties(field);
 
@@ -492,6 +494,12 @@ function _generateMappings(
               },
               isIndexModeTimeSeries
             );
+            if (field.object_type) {
+              // A group can have an object_type if it has been merged with an object during deduplication,
+              // generate also the dynamic mapping for the object.
+              addObjectAsDynamicMapping(field);
+              mappings.hasDynamicTemplateMappings = true;
+            }
             if (mappings.hasNonDynamicTemplateMappings) {
               fieldProps = {
                 properties: