Add support for ER with gRPC downstreams (#920)

Adds new gRPC annotation schemas, updates compatibility and validation logic.

CHANGELOG.md

@@ -14,6 +14,9 @@ and what APIs have changed, if applicable.
 
 ## [Unreleased]
 
+## [29.43.8] - 2023-07-13
+- Add support for gRPC-downstream extension annotations (`@grpcExtension`, `@grpcService`).
+
 ## [29.43.7] - 2023-07-11
 - Make file extension of D2 ZKFS file store fully customizable.
 
@@ -5499,7 +5502,8 @@ patch operations can re-use these classes for generating patch messages.
 
 ## [0.14.1]
 
-[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.43.7...master
+[Unreleased]: https://github.com/linkedin/rest.li/compare/v29.43.8...master
+[29.43.8]: https://github.com/linkedin/rest.li/compare/v29.43.7...v29.43.8
 [29.43.7]: https://github.com/linkedin/rest.li/compare/v29.43.6...v29.43.7
 [29.43.6]: https://github.com/linkedin/rest.li/compare/v29.43.5...v29.43.6
 [29.43.5]: https://github.com/linkedin/rest.li/compare/v29.43.4...v29.43.5

data/src/main/java/com/linkedin/data/schema/annotation/GrpcExtensionAnnotationHandler.java

@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) 2023 LinkedIn Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.linkedin.data.schema.annotation;
+
+import com.linkedin.data.DataMap;
+import com.linkedin.data.schema.PathSpec;
+import com.linkedin.data.schema.compatibility.CompatibilityMessage;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.lang3.tuple.Pair;
+
+
+/**
+ * This SchemaAnnotationHandler is used to check gRPC extension annotation(@grpcExtension) compatibility.
+ */
+public class GrpcExtensionAnnotationHandler implements SchemaAnnotationHandler
+{
+  public final static String GRPC_EXTENSION_ANNOTATION_NAMESPACE = "grpcExtension";
+
+  @Override
+  public ResolutionResult resolve(List<Pair<String, Object>> propertiesOverrides,
+      ResolutionMetaData resolutionMetadata)
+  {
+    // No-op, for extension schema there is no property resolve need.
+    return new ResolutionResult();
+  }
+
+  @Override
+  public String getAnnotationNamespace()
+  {
+    return GRPC_EXTENSION_ANNOTATION_NAMESPACE;
+  }
+
+  @Override
+  public AnnotationValidationResult validate(Map<String, Object> resolvedProperties, ValidationMetaData metaData)
+  {
+    // No-op, for extension schema there is no property resolve need, therefore there is no annotation validate need.
+    return new AnnotationValidationResult();
+  }
+
+  @Override
+  public SchemaVisitor getVisitor()
+  {
+    // No need to override properties, use IdentitySchemaVisitor to skip schema traverse.
+    return new IdentitySchemaVisitor();
+  }
+
+  @Override
+  public boolean implementsCheckCompatibility()
+  {
+    return true;
+  }
+
+  @Override
+  public AnnotationCompatibilityResult checkCompatibility(Map<String, Object> prevResolvedProperties, Map<String, Object> currResolvedProperties,
+      CompatibilityCheckContext prevContext, CompatibilityCheckContext currContext)
+  {
+    AnnotationCompatibilityResult result = new AnnotationCompatibilityResult();
+    // Both prevResolvedProperties and currResolvedProperties contain extension annotation namespace, check any changes of annotations on the existing fields.
+    if (prevResolvedProperties.containsKey(GRPC_EXTENSION_ANNOTATION_NAMESPACE) && currResolvedProperties.containsKey(GRPC_EXTENSION_ANNOTATION_NAMESPACE))
+    {
+      DataMap prevAnnotations = (DataMap) prevResolvedProperties.get(GRPC_EXTENSION_ANNOTATION_NAMESPACE);
+      DataMap currAnnotations = (DataMap) currResolvedProperties.get(GRPC_EXTENSION_ANNOTATION_NAMESPACE);
+      prevAnnotations.forEach((key, value) ->
+      {
+        if (currAnnotations.containsKey(key))
+        {
+          // Check annotation value changes.
+          if (!prevAnnotations.get(key).equals(currAnnotations.get(key)))
+          {
+            appendCompatibilityMessage(result, CompatibilityMessage.Impact.ANNOTATION_INCOMPATIBLE_CHANGE,
+                "Updating gRPC extension annotation field: \"%s\" value is considered a backward incompatible change.",
+                key, currContext.getPathSpecToSchema());
+          }
+          currAnnotations.remove(key);
+        }
+        else
+        {
+          // An existing annotation field is removed.
+          appendCompatibilityMessage(result, CompatibilityMessage.Impact.ANNOTATION_INCOMPATIBLE_CHANGE,
+              "Removing gRPC extension annotation field: \"%s\" is considered an backward incompatible change.",
+              key, currContext.getPathSpecToSchema());
+        }
+      });
+
+      currAnnotations.forEach((key, value) ->
+      {
+        // Adding an extension annotation field.
+        appendCompatibilityMessage(result, CompatibilityMessage.Impact.ANNOTATION_INCOMPATIBLE_CHANGE,
+            "Adding gRPC extension annotation field: \"%s\" is a backward incompatible change.",
+            key, currContext.getPathSpecToSchema());
+      });
+    }
+    else if (prevResolvedProperties.containsKey(GRPC_EXTENSION_ANNOTATION_NAMESPACE))
+    {
+      // Only previous schema has extension annotation, it means the extension annotation is removed in the current schema.
+      if (currContext.getPathSpecToSchema() != null)
+      {
+        appendCompatibilityMessage(result, CompatibilityMessage.Impact.ANNOTATION_INCOMPATIBLE_CHANGE,
+            "Removing gRPC extension annotation is a backward incompatible change.",
+            null, prevContext.getPathSpecToSchema());
+      }
+      else
+      {
+        // an existing field with extension annotation is removed
+        appendCompatibilityMessage(result, CompatibilityMessage.Impact.ANNOTATION_INCOMPATIBLE_CHANGE,
+            "Removing field: \"%s\" with gRPC extension annotation is a backward incompatible change.",
+            prevContext.getSchemaField().getName(), prevContext.getPathSpecToSchema());
+      }
+    }
+    else
+    {
+      if (prevContext.getPathSpecToSchema() != null)
+      {
+        appendCompatibilityMessage(result, CompatibilityMessage.Impact.ANNOTATION_INCOMPATIBLE_CHANGE,
+            "Adding gRPC extension annotation on an existing field: \"%s\" is backward incompatible change",
+            prevContext.getSchemaField().getName() , currContext.getPathSpecToSchema());
+      }
+      else
+      {
+        // Adding a new injected field with extension annotation.
+        appendCompatibilityMessage(result, CompatibilityMessage.Impact.ANNOTATION_COMPATIBLE_CHANGE,
+            "Adding gRPC extension annotation on new field: \"%s\" is backward compatible change", currContext.getSchemaField().getName() , currContext.getPathSpecToSchema());
+      }
+    }
+    return result;
+  }
+
+  private void appendCompatibilityMessage(AnnotationCompatibilityResult result, CompatibilityMessage.Impact impact, String message, String context, PathSpec pathSpec)
+  {
+    CompatibilityMessage compatibilityMessage = new CompatibilityMessage(pathSpec, impact, message, context);
+    result.addMessage(compatibilityMessage);
+  }
+}

gradle.properties

@@ -1,4 +1,4 @@
-version=29.43.7
+version=29.43.8
 group=com.linkedin.pegasus
 org.gradle.configureondemand=true
 org.gradle.parallel=true

restli-common/src/main/pegasus/com/linkedin/restli/common/GrpcExtensionAnnotation.pdl

@@ -0,0 +1,30 @@
+namespace com.linkedin.restli.common
+
+/**
+ * Specifies the extension schema field annotation format for gRPC downstreams.
+ */
+record GrpcExtensionAnnotation {
+
+  /**
+   * The RPC method used for this injection.
+   * For 1-to-many relationships, can use either GET_ALL or FINDER.
+   * For 1-to-1 relationships, it must be omitted for collection resources or use GET for simple resources.
+   */
+  rpc: optional string
+
+  /**
+   * How to construct the RPC message in the injection request for 1-to-many relations.
+   */
+  params: optional map[string, string]
+
+  /**
+   * Used to specify the injected URN's parts so that it may be reconstructed and its resolver can be used.
+   * For 1-to-1 relationships, the injected URN resolver is needed so that the injected entity can be fetched.
+   */
+  injectedUrnParts: optional map[string, string]
+
+  /**
+   * Specifies versionSuffix in multi-version scenario. If is is not provided, will pick first version by default.
+   */
+  versionSuffix: optional string
+}

restli-common/src/main/resources/legacyPegasusSchemas/com/linkedin/restli/common/GrpcExtensionAnnotation.pdsc

@@ -0,0 +1,33 @@
+{
+  "type" : "record",
+  "name" : "GrpcExtensionAnnotation",
+  "namespace" : "com.linkedin.restli.common",
+  "doc" : "Specifies the extension schema field annotation format for gRPC downstreams.",
+  "fields" : [ {
+    "name" : "rpc",
+    "type" : "string",
+    "doc" : "The RPC method used for this injection.\nFor 1-to-many relationships, can use either GET_ALL or FINDER.\nFor 1-to-1 relationships, it must be omitted for collection resources or use GET for simple resources.",
+    "optional" : true
+  }, {
+    "name" : "params",
+    "type" : {
+      "type" : "map",
+      "values" : "string"
+    },
+    "doc" : "How to construct the RPC message in the injection request for 1-to-many relations.",
+    "optional" : true
+  }, {
+    "name" : "injectedUrnParts",
+    "type" : {
+      "type" : "map",
+      "values" : "string"
+    },
+    "doc" : "Used to specify the injected URN's parts so that it may be reconstructed and its resolver can be used.\nFor 1-to-1 relationships, the injected URN resolver is needed so that the injected entity can be fetched.",
+    "optional" : true
+  }, {
+    "name" : "versionSuffix",
+    "type" : "string",
+    "doc" : "Specifies versionSuffix in multi-version scenario. If is is not provided, will pick first version by default.",
+    "optional" : true
+  } ]
+}