Skip to content

Commit

Permalink
Merge pull request #73 from booddu/master
Browse files Browse the repository at this point in the history 
Adding capability on whether RECORD type can be explicitly set as nul…
  • Loading branch information
@booddu
booddu authored Aug 30, 2022
2 parents 30ee971 + 88a8f83 commit b4c38cf
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 5 deletions.
28 changes: 23 additions & 5 deletions cdi-core/src/main/java/com/linkedin/cdi/util/SchemaBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -470,6 +470,22 @@ public JsonElement buildAltSchema() {
return buildAltSchema(new HashMap<>(), false, null, null, false);
}

public JsonElement buildAltSchema(boolean recordTypeNullable) {
return buildAltSchema(new HashMap<>(), false, null, null, false, recordTypeNullable);
}

/**
* For backward compatibility.
* i.e. no explicit nullable setting for sub-tables (RECORD type)
*/
public JsonElement buildAltSchema(Map<String, String> defaultTypes,
boolean enableCleansing,
String pattern,
String replacement,
boolean nullable) {
return buildAltSchema(defaultTypes, enableCleansing, pattern, replacement, nullable, false);
}

/**
* Build into a Avro flavored, but not true Avro, schema that can be fed into
* Json2Avro converter. Along the way, schema names are cleansed if special characters
Expand All @@ -486,19 +502,21 @@ public JsonElement buildAltSchema() {
* @param pattern the search pattern of schema cleansing
* @param replacement the replacement string for schema cleansing
* @param nullable whether to force output all columns as nullable
* @param recordTypeNullable whether RECORD type can be explicitly set as nullable
* @return the Avro flavored schema definition
*/
public JsonElement buildAltSchema(Map<String, String> defaultTypes,
boolean enableCleansing,
String pattern,
String replacement,
boolean nullable) {
boolean nullable,
boolean recordTypeNullable) {
JsonObject nestedType = new JsonObject();
if (this.type == RECORD
|| (this.type == ARRAY && this.elements.size() > 1)) {
JsonArray fields = new JsonArray();
for (SchemaBuilder field : elements) {
fields.add(field.buildAltSchema(defaultTypes, enableCleansing, pattern, replacement, nullable));
fields.add(field.buildAltSchema(defaultTypes, enableCleansing, pattern, replacement, nullable, recordTypeNullable));
}
if (name.equals("root") || type == ARRAY) {
return fields;
Expand All @@ -513,16 +531,16 @@ public JsonElement buildAltSchema(Map<String, String> defaultTypes,
nestedType.addProperty(KEY_WORD_ITEMS, this.elements.get(0).getPrimitiveType());
} else {
nestedType.add(KEY_WORD_ITEMS,
this.elements.get(0).buildAltSchema(defaultTypes, enableCleansing, pattern, replacement, nullable));
this.elements.get(0).buildAltSchema(defaultTypes, enableCleansing, pattern, replacement, nullable, recordTypeNullable));
}
} else {
nestedType.addProperty(KEY_WORD_TYPE, defaultTypes.getOrDefault(this.getName(), this.primitiveType));
}

JsonObject column = new JsonObject();
column.addProperty(KEY_WORD_COLUMN_NAME, enableCleansing ? name.replaceAll(pattern, replacement) : name);
// no explicit nullable setting for sub-tables
if (this.type != RECORD) {
// no explicit nullable setting for sub-tables, unless explicitly requested
if (this.type != RECORD || recordTypeNullable) {
column.addProperty(KEY_WORD_DATA_IS_NULLABLE, nullable || this.isNullable);
}
column.add(KEY_WORD_DATA_TYPE, nestedType);
Expand Down
16 changes: 16 additions & 0 deletions cdi-core/src/test/java/com/linkedin/cdi/util/SchemaBuilderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

package com.linkedin.cdi.util;

import com.google.gson.Gson;
import com.google.gson.JsonObject;
import java.util.HashMap;
import org.testng.Assert;
import org.testng.annotations.Test;
Expand Down Expand Up @@ -45,4 +47,18 @@ public void testAltSchema() {
Assert.assertEquals(builder.buildAltSchema().toString(),
"[{\"columnName\":\"methods\",\"isNullable\":false,\"dataType\":{\"type\":\"array\",\"name\":\"methods\",\"items\":\"string\"}}]");
}

@Test
public void testAltSchemaForRecord() {
String avroSchema = "{\"type\":\"record\",\"name\":\"UploadClickConversionsResponse\",\"namespace\":\"com.linkedin.coderising\",\"doc\":\"Response for Ads\\nhttps://developers.google.com/google-ads/api/reference/rpc/v6/UploadClickConversionsResponse\",\"fields\":[{\"name\":\"partialFailureError\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"partialFailureErrorRecord\",\"fields\":[{\"name\":\"code\",\"type\":[\"null\",\"int\"],\"doc\":\"code of the faliure\",\"default\":null,\"compliance\":\"NONE\"},{\"name\":\"message\",\"type\":[\"null\",\"string\"],\"doc\":\"message of the failure\",\"default\":null,\"compliance\":\"NONE\"},{\"name\":\"details\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"detailRecord\",\"doc\":\"detail records\",\"fields\":[{\"name\":\"_type\",\"type\":[\"null\",\"string\"],\"default\":null,\"compliance\":\"NONE\"},{\"name\":\"errors\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"errorRecord\",\"fields\":[{\"name\":\"errorCode\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"errorCodeRecord\",\"fields\":[{\"name\":\"conversionUploadError\",\"type\":[\"null\",\"string\"],\"default\":null,\"compliance\":\"NONE\"}]}],\"default\":null},{\"name\":\"message\",\"type\":[\"null\",\"string\"],\"default\":null,\"compliance\":\"NONE\"},{\"name\":\"trigger\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"triggerRecord\",\"fields\":[{\"name\":\"stringValue\",\"type\":[\"null\",\"string\"],\"default\":null,\"compliance\":\"NONE\"}]}],\"default\":null},{\"name\":\"location\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"locationRecord\",\"fields\":[{\"name\":\"fieldPathElements\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"fieldPathElementRecord\",\"fields\":[{\"name\":\"fieldName\",\"type\":[\"null\",\"string\"],\"default\":null,\"compliance\":\"NONE\"},{\"name\":\"index\",\"type\":[\"null\",\"int\"],\"default\":null,\"compliance\":\"NONE\"}]}}],\"default\":null}]}],\"default\":null}]}}],\"default\":null}]}}],\"doc\":\"details of the failure\",\"default\":null}]}],\"doc\":\"partial failure error\",\"default\":null},{\"name\":\"results\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"resultRecord\",\"fields\":[{\"name\":\"gclid\",\"type\":[\"null\",\"string\"],\"default\":null,\"compliance\":\"NONE\"},{\"name\":\"conversionAction\",\"type\":[\"null\",\"string\"],\"default\":null,\"compliance\":\"NONE\"},{\"name\":\"conversionDateTime\",\"type\":[\"null\",\"string\"],\"default\":null,\"compliance\":\"NONE\"}]}}],\"doc\":\"Results\",\"default\":null}],\"collection\":{\"name\":\"UploadClickConversionsResponse\"}}";
JsonObject schema = new Gson().fromJson(avroSchema, JsonObject.class);
SchemaBuilder builder = SchemaBuilder.fromAvroSchema(schema);

Assert.assertEquals(builder.buildAltSchema().toString(),
"[{\"columnName\":\"partialFailureError\",\"dataType\":{\"type\":\"record\",\"name\":\"partialFailureError\",\"values\":[{\"columnName\":\"code\",\"isNullable\":true,\"dataType\":{\"type\":\"int\"}},{\"columnName\":\"message\",\"isNullable\":true,\"dataType\":{\"type\":\"string\"}},{\"columnName\":\"details\",\"isNullable\":true,\"dataType\":{\"type\":\"array\",\"name\":\"details\",\"items\":{\"columnName\":\"arrayItem\",\"dataType\":{\"type\":\"record\",\"name\":\"arrayItem\",\"values\":[{\"columnName\":\"_type\",\"isNullable\":true,\"dataType\":{\"type\":\"string\"}},{\"columnName\":\"errors\",\"isNullable\":true,\"dataType\":{\"type\":\"array\",\"name\":\"errors\",\"items\":{\"columnName\":\"arrayItem\",\"dataType\":{\"type\":\"record\",\"name\":\"arrayItem\",\"values\":[{\"columnName\":\"errorCode\",\"dataType\":{\"type\":\"record\",\"name\":\"errorCode\",\"values\":[{\"columnName\":\"conversionUploadError\",\"isNullable\":true,\"dataType\":{\"type\":\"string\"}}]}},{\"columnName\":\"message\",\"isNullable\":true,\"dataType\":{\"type\":\"string\"}},{\"columnName\":\"trigger\",\"dataType\":{\"type\":\"record\",\"name\":\"trigger\",\"values\":[{\"columnName\":\"stringValue\",\"isNullable\":true,\"dataType\":{\"type\":\"string\"}}]}},{\"columnName\":\"location\",\"dataType\":{\"type\":\"record\",\"name\":\"location\",\"values\":[{\"columnName\":\"fieldPathElements\",\"isNullable\":true,\"dataType\":{\"type\":\"array\",\"name\":\"fieldPathElements\",\"items\":{\"columnName\":\"arrayItem\",\"dataType\":{\"type\":\"record\",\"name\":\"arrayItem\",\"values\":[{\"columnName\":\"fieldName\",\"isNullable\":true,\"dataType\":{\"type\":\"string\"}},{\"columnName\":\"index\",\"isNullable\":true,\"dataType\":{\"type\":\"int\"}}]}}}}]}}]}}}}]}}}}]}},{\"columnName\":\"results\",\"isNullable\":true,\"dataType\":{\"type\":\"array\",\"name\":\"results\",\"items\":{\"columnName\":\"arrayItem\",\"dataType\":{\"type\":\"record\",\"name\":\"arrayItem\",\"values\":[{\"columnName\":\"gclid\",\"isNullable\":true,\"dataType\":{\"type\":\"string\"}},{\"columnName\":\"conversionAction\",\"isNullable\":true,\"dataType\":{\"type\":\"string\"}},{\"columnName\":\"conversionDateTime\",\"isNullable\":true,\"dataType\":{\"type\":\"string\"}}]}}}}]");

Assert.assertEquals(builder.buildAltSchema(true).toString(),
"[{\"columnName\":\"partialFailureError\",\"isNullable\":true,\"dataType\":{\"type\":\"record\",\"name\":\"partialFailureError\",\"values\":[{\"columnName\":\"code\",\"isNullable\":true,\"dataType\":{\"type\":\"int\"}},{\"columnName\":\"message\",\"isNullable\":true,\"dataType\":{\"type\":\"string\"}},{\"columnName\":\"details\",\"isNullable\":true,\"dataType\":{\"type\":\"array\",\"name\":\"details\",\"items\":{\"columnName\":\"arrayItem\",\"isNullable\":false,\"dataType\":{\"type\":\"record\",\"name\":\"arrayItem\",\"values\":[{\"columnName\":\"_type\",\"isNullable\":true,\"dataType\":{\"type\":\"string\"}},{\"columnName\":\"errors\",\"isNullable\":true,\"dataType\":{\"type\":\"array\",\"name\":\"errors\",\"items\":{\"columnName\":\"arrayItem\",\"isNullable\":false,\"dataType\":{\"type\":\"record\",\"name\":\"arrayItem\",\"values\":[{\"columnName\":\"errorCode\",\"isNullable\":true,\"dataType\":{\"type\":\"record\",\"name\":\"errorCode\",\"values\":[{\"columnName\":\"conversionUploadError\",\"isNullable\":true,\"dataType\":{\"type\":\"string\"}}]}},{\"columnName\":\"message\",\"isNullable\":true,\"dataType\":{\"type\":\"string\"}},{\"columnName\":\"trigger\",\"isNullable\":true,\"dataType\":{\"type\":\"record\",\"name\":\"trigger\",\"values\":[{\"columnName\":\"stringValue\",\"isNullable\":true,\"dataType\":{\"type\":\"string\"}}]}},{\"columnName\":\"location\",\"isNullable\":true,\"dataType\":{\"type\":\"record\",\"name\":\"location\",\"values\":[{\"columnName\":\"fieldPathElements\",\"isNullable\":true,\"dataType\":{\"type\":\"array\",\"name\":\"fieldPathElements\",\"items\":{\"columnName\":\"arrayItem\",\"isNullable\":false,\"dataType\":{\"type\":\"record\",\"name\":\"arrayItem\",\"values\":[{\"columnName\":\"fieldName\",\"isNullable\":true,\"dataType\":{\"type\":\"string\"}},{\"columnName\":\"index\",\"isNullable\":true,\"dataType\":{\"type\":\"int\"}}]}}}}]}}]}}}}]}}}}]}},{\"columnName\":\"results\",\"isNullable\":true,\"dataType\":{\"type\":\"array\",\"name\":\"results\",\"items\":{\"columnName\":\"arrayItem\",\"isNullable\":false,\"dataType\":{\"type\":\"record\",\"name\":\"arrayItem\",\"values\":[{\"columnName\":\"gclid\",\"isNullable\":true,\"dataType\":{\"type\":\"string\"}},{\"columnName\":\"conversionAction\",\"isNullable\":true,\"dataType\":{\"type\":\"string\"}},{\"columnName\":\"conversionDateTime\",\"isNullable\":true,\"dataType\":{\"type\":\"string\"}}]}}}}]");

}
}

0 comments on commit b4c38cf

Please sign in to comment.