From 9ff6254552829746cbb3d8e20fc1f41b4ad45a5f Mon Sep 17 00:00:00 2001 From: markbwarner Date: Thu, 15 Aug 2024 11:08:22 -0400 Subject: [PATCH] Snowflake CRDP functions and better error handling --- .../python/ThalesGCPSnowCTVLTokenize.py | 99 --- .../python/requirements.txt | 5 - .../ThalesGCPSnowCADPCharDecryptBulkFPE.java | 222 ------ .../ThalesGCPSnowCADPCharDecryptFPE.java | 147 ---- .../ThalesGCPSnowCADPCharEncryptBulkFPE.java | 219 ------ .../ThalesGCPSnowCADPCharEncryptFPE.java | 144 ---- .../ThalesGCPSnowCADPNbrDecryptBulkFPE.java | 218 ------ .../ThalesGCPSnowCADPNbrDecryptFPE.java | 146 ---- .../ThalesGCPSnowCADPNbrEncryptBulkFPE.java | 220 ------ .../ThalesGCPSnowCADPNbrEncryptFPE.java | 144 ---- .../main/resources/CADP_for_JAVA.properties | 657 ----------------- .../src/test/java/GCPCloudFunctionTester.java | 173 ----- database/snowflake/README.md | 6 +- .../snowflake/Thales-Snow-AWS-UDF/pom.xml | 147 ++++ .../main/java/example/CMUserSetHelper.java | 509 +++++++++++++ .../main/java/example/CustomException.java | 15 + .../example/ThalesAWSSnowCADPFPEBulkUDF.java | 511 +++++++++++++ .../java/example/ThalesAWSSnowCADPFPEUDF.java | 348 +++++++++ .../example/ThalesAWSSnowCRDPFPEBulkUDF.java | 674 +++++++++++++++++ .../java/example/ThalesAWSSnowCRDPFPEUDF.java | 430 +++++++++++ .../src/test/java/CMUserSetHelper.java | 509 +++++++++++++ .../src/test/java/CustomException.java | 15 + .../ThalesAWSSnowCADPFPEBulkUDFTester.java | 631 ++++++++++++++++ .../java/ThalesAWSSnowCADPFPEUDFTester.java | 433 +++++++++++ .../ThalesAWSSnowCRDPFPEBulkUDFTester.java | 682 ++++++++++++++++++ .../java/ThalesAWSSnowCRDPFPEUDFTester.java | 515 +++++++++++++ .../pom.xml | 11 +- .../java/com/example/CMUserSetHelper.java | 508 +++++++++++++ .../java/com/example/CustomException.java | 15 + .../com/example/ThalesGCPSnowCADPBulkFPE.java | 493 +++++++++++++ .../com/example/ThalesGCPSnowCADPFPE.java | 308 ++++++++ .../com/example/ThalesGCPSnowCRDPBulkFPE.java | 523 ++++++++++++++ .../com/example/ThalesGCPSnowCRDPFPE.java | 371 ++++++++++ .../src/test/java/CMUserSetHelper.java | 508 +++++++++++++ .../src/test/java/CustomException.java | 15 + .../java/ThalesGCPSnowCADPBulkFPETester.java | 525 ++++++++++++++ .../test/java/ThalesGCPSnowCADPFPETester.java | 337 +++++++++ .../java/ThalesGCPSnowCRDPBulkFPETester.java | 568 +++++++++++++++ .../test/java/ThalesGCPSnowCRDPFPETester.java | 415 +++++++++++ 39 files changed, 10011 insertions(+), 2405 deletions(-) delete mode 100644 database/snowflake/CADP-SNOW-GCP-Functions/python/ThalesGCPSnowCTVLTokenize.py delete mode 100644 database/snowflake/CADP-SNOW-GCP-Functions/python/requirements.txt delete mode 100644 database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharDecryptBulkFPE.java delete mode 100644 database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharDecryptFPE.java delete mode 100644 database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharEncryptBulkFPE.java delete mode 100644 database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharEncryptFPE.java delete mode 100644 database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrDecryptBulkFPE.java delete mode 100644 database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrDecryptFPE.java delete mode 100644 database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrEncryptBulkFPE.java delete mode 100644 database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrEncryptFPE.java delete mode 100644 database/snowflake/CADP-SNOW-GCP-Functions/src/main/resources/CADP_for_JAVA.properties delete mode 100644 database/snowflake/CADP-SNOW-GCP-Functions/src/test/java/GCPCloudFunctionTester.java create mode 100644 database/snowflake/Thales-Snow-AWS-UDF/pom.xml create mode 100644 database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/CMUserSetHelper.java create mode 100644 database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/CustomException.java create mode 100644 database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCADPFPEBulkUDF.java create mode 100644 database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCADPFPEUDF.java create mode 100644 database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCRDPFPEBulkUDF.java create mode 100644 database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCRDPFPEUDF.java create mode 100644 database/snowflake/Thales-Snow-AWS-UDF/src/test/java/CMUserSetHelper.java create mode 100644 database/snowflake/Thales-Snow-AWS-UDF/src/test/java/CustomException.java create mode 100644 database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCADPFPEBulkUDFTester.java create mode 100644 database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCADPFPEUDFTester.java create mode 100644 database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCRDPFPEBulkUDFTester.java create mode 100644 database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCRDPFPEUDFTester.java rename database/snowflake/{CADP-SNOW-GCP-Functions => Thales-Snow-GCP-UDF}/pom.xml (94%) create mode 100644 database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/CMUserSetHelper.java create mode 100644 database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/CustomException.java create mode 100644 database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCADPBulkFPE.java create mode 100644 database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCADPFPE.java create mode 100644 database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCRDPBulkFPE.java create mode 100644 database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCRDPFPE.java create mode 100644 database/snowflake/Thales-Snow-GCP-UDF/src/test/java/CMUserSetHelper.java create mode 100644 database/snowflake/Thales-Snow-GCP-UDF/src/test/java/CustomException.java create mode 100644 database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCADPBulkFPETester.java create mode 100644 database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCADPFPETester.java create mode 100644 database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCRDPBulkFPETester.java create mode 100644 database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCRDPFPETester.java diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/python/ThalesGCPSnowCTVLTokenize.py b/database/snowflake/CADP-SNOW-GCP-Functions/python/ThalesGCPSnowCTVLTokenize.py deleted file mode 100644 index d5d616e8..00000000 --- a/database/snowflake/CADP-SNOW-GCP-Functions/python/ThalesGCPSnowCTVLTokenize.py +++ /dev/null @@ -1,99 +0,0 @@ -# This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It uses CT-VL -# to protect sensitive data in a column. This example will tokenize the data. -# -# Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested -# for all possible data sizes and combinations of encryption algorithms and IV, etc. -# Was tested with CM 2.11 & CT-VL 2.6 or higher. -# For more information on Snowflake External Functions see link below. -# https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp -# -# @author mwarner -# -# - - -import sys -import requests -import json -import os -#from requests.packages.urllib3.exceptions import InsecureRequestWarning -from urllib3 import disable_warnings -from urllib3.exceptions import InsecureRequestWarning -disable_warnings(InsecureRequestWarning) - - -HTTP_SUCCESS = 200 -HTTP_FAILURE = 400 -# Declare the variable - -p11ptext = None -p11auth = None - -p11debug = True -p11user = os.environ.get('P11USER') -p11url = os.environ.get('P11URL') -p11tokgroup = "tg1" -p11toktemplate = "tt1" -p11ptext = "thisistestdata" -# Loop through the option, set value to variable - -p11url = "https://" + p11url + "/vts" -# Check username and password if exist in system environment or not - -# split the p11user variable to check username/password missing -p11auth = tuple(p11user.split(":", 1)) - -errs = 0 - -# Set the header application -hdrs = {'Content-Type': 'application/json'} -requests.packages.urllib3.disable_warnings(InsecureRequestWarning) - -############ Check connect to VTS before continue ####################### -try: - r = requests.get(p11url, verify=False) - r.raise_for_status() -except requests.exceptions.RequestException as err: - print("\nEror, something wrong: ") - print(err) - sys.exit(200) -except requests.exceptions.HTTPError as errh: - print("\nHttp Error:", errh) - sys.exit(200) -except requests.exceptions.ConnectionError as errc: - print("\nError Connecting:", errc) - sys.exit(200) -except requests.exceptions.Timeout as errt: - print("\nTimeout Error:", errt) - sys.exit(200) - -################################################################################ - -def thales_cts_tokenize(request): - - try: - # The list of rows to return. - return_value = [] - result = None - payload = request.get_json() - - rows = payload["data"] - u = p11url + "/rest/v2.0/tokenize" - # For each input row - for row in rows: - # Include the row number. - row_number = row[0] - row_value = row[1] - data = {"tokengroup" : p11tokgroup, "tokentemplate" : p11toktemplate} - data["data"] = row_value - # Post the request - r = requests.post(u, headers=hdrs, auth=p11auth, verify=False, json=data) - result = json.loads(r.text) - row_to_return = [row_number, result['token']] - return_value.append(row_to_return) - json_compatible_string_to_return = json.dumps( { "data" : return_value } ) - return (json_compatible_string_to_return, HTTP_SUCCESS) - - except: - - return(request.data, HTTP_FAILURE) \ No newline at end of file diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/python/requirements.txt b/database/snowflake/CADP-SNOW-GCP-Functions/python/requirements.txt deleted file mode 100644 index 6dc34932..00000000 --- a/database/snowflake/CADP-SNOW-GCP-Functions/python/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Function dependencies, for example: -# package>=version -functions-framework -urllib3==1.26.18 -requests==2.32.2 \ No newline at end of file diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharDecryptBulkFPE.java b/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharDecryptBulkFPE.java deleted file mode 100644 index 35e24eb6..00000000 --- a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharDecryptBulkFPE.java +++ /dev/null @@ -1,222 +0,0 @@ -package com.example; - -import javax.crypto.Cipher; -import com.google.cloud.functions.HttpFunction; -import com.google.cloud.functions.HttpRequest; -import com.google.cloud.functions.HttpResponse; -import com.ingrian.security.nae.IngrianProvider; -import com.ingrian.security.nae.AbstractNAECipher; -import com.ingrian.security.nae.FPEParameterAndFormatSpec; -import com.ingrian.security.nae.NAEKey; -import com.ingrian.security.nae.NAESession; -import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder; -import com.ingrian.security.nae.IngrianProvider.Builder; -import com.ingrian.security.nae.NAECipher; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonPrimitive; - -import java.security.spec.AlgorithmParameterSpec; -import java.util.HashMap; -import java.util.Map; -import java.util.logging.Logger; - -/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application - * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the - * data so applications or business intelligence tools do not have to change in order to use these columns. -* -* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested -* for all possible data sizes and combinations of encryption algorithms and IV, etc. -* Was tested with CM 2.11 & CADP 8.13 -* For more information on CADP see link below. -https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html -* For more information on Snowflake External Functions see link below. -https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp - -Notes: This example uses the CADP bulk API. -Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data -element has Unicode characters then the supported size limit would be 1750 characters - -Size of spec array should be same as the number of elements if user wants to use separate spec values -for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data -index. - * - *@author mwarner - * - */ - -public class ThalesGCPSnowCADPCharDecryptBulkFPE implements HttpFunction { -// @Override - private static byte[][] data; - private static AlgorithmParameterSpec[] spec; - private static Integer[] snowrownbrs; - - private static int BATCHLIMIT = 10000; - private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPCharDecryptBulkFPE.class.getName()); - private static final Gson gson = new Gson(); - - public void service(HttpRequest request, HttpResponse response) throws Exception { - - String tweakAlgo = null; - String tweakData = null; - String snowflakereturnstring = null; - JsonArray snowflakedata = null; - NAESession session = null; - - Map encryptedErrorMapTotal = new HashMap(); - try { - - String keyName = "testfaas"; - String userName = System.getenv("CMUSER"); - String password = System.getenv("CMPWD"); - int batchsize = Integer.parseInt(System.getenv("BATCHSIZE")); - // JsonArray snowflakedata = null; - - if (batchsize >= BATCHLIMIT) - batchsize = BATCHLIMIT; - spec = new FPEParameterAndFormatSpec[batchsize]; - data = new byte[batchsize][]; - snowrownbrs = new Integer[batchsize]; - - try { - JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class); - JsonObject requestJson = null; - - if (requestParsed != null && requestParsed.isJsonObject()) { - requestJson = requestParsed.getAsJsonObject(); - } - - if (requestJson != null && requestJson.has("data")) { - snowflakedata = requestJson.getAsJsonArray("data"); - - } - - } catch (JsonParseException e) { - logger.severe("Error parsing JSON: " + e.getMessage()); - } - - // System.setProperty("com.ingrian.security.nae.IngrianNAE_Properties_Conf_Filename", - // "IngrianNAE.properties"); - System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", - "CADP_for_JAVA.properties"); - IngrianProvider builder = new Builder().addConfigFileInputStream( - getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build(); - // IngrianProvider builder = new - // Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build(); - session = NAESession.getSession(userName, password.toCharArray()); - NAEKey key = NAEKey.getSecretKey(keyName, session); - int row_number = 0; - - StringBuffer snowflakereturndata = new StringBuffer(); - // Serialization - snowflakereturndata.append("{ \"data\": ["); - String algorithm = "FPE/FF1/CARD62"; - - AbstractNAECipher decryptCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider"); - int i = 0; - - int totalRowsLeft = snowflakedata.size(); - while (i < snowflakedata.size()) { - int index = 0; - int dataIndex = 0; - int specIndex = 0; - int snowRowIndex = 0; - // System.out.println("totalRowsLeft begining =" + totalRowsLeft); - if (totalRowsLeft < batchsize) { - spec = new FPEParameterAndFormatSpec[totalRowsLeft]; - data = new byte[totalRowsLeft][]; - snowrownbrs = new Integer[totalRowsLeft]; - } else { - spec = new FPEParameterAndFormatSpec[batchsize]; - data = new byte[batchsize][]; - snowrownbrs = new Integer[batchsize]; - } - - for (int b = 0; b < batchsize && b < totalRowsLeft; b++) { - - JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); - - for (int j = 0; j < snowflakerow.size(); j++) { - - JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); - // System.out.print(snowflakecolumn + " "); - String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); - // get a cipher - if (j == 1) { - - // FPE example - data[dataIndex++] = sensitive.getBytes(); - spec[specIndex++] = new FPEParameterAndFormatBuilder(tweakData) - .set_tweakAlgorithm(tweakAlgo).build(); - } else { - - row_number = snowflakecolumn.getAsInt(); - snowrownbrs[snowRowIndex++] = row_number; - - } - - } - - i++; - } - // make bulk call.... - // initializing the cipher for encrypt operation - decryptCipher.init(Cipher.DECRYPT_MODE, key, spec[0]); - - // Map to store exceptions while encryption - Map decryptErrorMap = new HashMap(); - - // performing bulk operation - byte[][] decryptedData = decryptCipher.doFinalBulk(data, spec, decryptErrorMap); - - for (Map.Entry entry : decryptErrorMap.entrySet()) { - Integer mkey = entry.getKey(); - String mvalue = entry.getValue(); - encryptedErrorMapTotal.put(mkey, mvalue); - } - - for (int enc = 0; enc < decryptedData.length; enc++) { - - snowflakereturndata.append("["); - snowflakereturndata.append(snowrownbrs[enc]); - snowflakereturndata.append(","); - snowflakereturndata.append(new String(decryptedData[enc])); - - if (index <= batchsize - 1 || index < totalRowsLeft - 1) - // if (index < batchsize -1 && index < totalRowsLeft -1 ) - { - if (totalRowsLeft - 1 > 0) - snowflakereturndata.append("],"); - else - snowflakereturndata.append("]"); - } else { - - snowflakereturndata.append("]"); - } - - index++; - totalRowsLeft--; - - } - } - - snowflakereturndata.append("]}"); - - snowflakereturnstring = new String(snowflakereturndata); - - } catch (Exception e) { - // return "check exception"; - } - finally{ - if(session!=null) { - session.closeSession(); - } - } - - response.getWriter().write(snowflakereturnstring); - - } -} \ No newline at end of file diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharDecryptFPE.java b/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharDecryptFPE.java deleted file mode 100644 index 4692b1e4..00000000 --- a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharDecryptFPE.java +++ /dev/null @@ -1,147 +0,0 @@ -package com.example; - -import javax.crypto.Cipher; -import com.google.cloud.functions.HttpFunction; -import com.google.cloud.functions.HttpRequest; -import com.google.cloud.functions.HttpResponse; -import com.ingrian.security.nae.IngrianProvider; -import com.ingrian.security.nae.FPEParameterAndFormatSpec; -import com.ingrian.security.nae.NAEKey; -import com.ingrian.security.nae.NAESession; -import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder; -import com.ingrian.security.nae.IngrianProvider.Builder; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonPrimitive; -import java.util.logging.Logger; - -/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application - * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the - * data so applications or business intelligence tools do not have to change in order to use these columns. -* -* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested -* for all possible data sizes and combinations of encryption algorithms and IV, etc. -* Was tested with CM 2.11 & CADP 8.13 -* For more information on CADP see link below. -https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html -* For more information on Snowflake External Functions see link below. -https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp - * - *@author mwarner - * - */ - -public class ThalesGCPSnowCADPCharDecryptFPE implements HttpFunction { -// @Override - - private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPCharDecryptFPE.class.getName()); - private static final Gson gson = new Gson(); - - public void service(HttpRequest request, HttpResponse response) throws Exception { - - String decData = ""; - String tweakAlgo = null; - String tweakData = null; - String snowflakereturnstring = null; - NAESession session = null; - - try { - - String keyName = "testfaas"; - String userName = System.getenv("CMUSER"); - String password = System.getenv("CMPWD"); - JsonArray snowflakedata = null; - - try { - JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class); - JsonObject requestJson = null; - - if (requestParsed != null && requestParsed.isJsonObject()) { - requestJson = requestParsed.getAsJsonObject(); - } - - if (requestJson != null && requestJson.has("data")) { - snowflakedata = requestJson.getAsJsonArray("data"); - - } - - } catch (JsonParseException e) { - logger.severe("Error parsing JSON: " + e.getMessage()); - } - - // System.setProperty("com.ingrian.security.nae.IngrianNAE_Properties_Conf_Filename", - // "IngrianNAE.properties"); - System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", - "CADP_for_JAVA.properties"); - IngrianProvider builder = new Builder().addConfigFileInputStream( - getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build(); - // IngrianProvider builder = new - // Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build(); - session = NAESession.getSession(userName, password.toCharArray()); - NAEKey key = NAEKey.getSecretKey(keyName, session); - int row_number = 0; - - StringBuffer snowflakereturndata = new StringBuffer(); - // Serialization - snowflakereturndata.append("{ \"data\": ["); - String algorithm = "FPE/FF1/CARD62"; - // String algorithm = "AES/CBC/PKCS5Padding"; - FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo) - .build(); - - Cipher decryptCipher = Cipher.getInstance(algorithm, "IngrianProvider"); - - for (int i = 0; i < snowflakedata.size(); i++) { - JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); - snowflakereturndata.append("["); - for (int j = 0; j < snowflakerow.size(); j++) { - - JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); - String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); - // get a cipher - if (j == 1) { - - // FPE example - - // initialize cipher to encrypt. - decryptCipher.init(Cipher.DECRYPT_MODE, key, param); - // encrypt data - byte[] outbuf = decryptCipher.doFinal(sensitive.getBytes()); - decData = new String(outbuf); - - snowflakereturndata.append(decData); - - } else { - row_number = snowflakecolumn.getAsInt(); - snowflakereturndata.append(row_number); - snowflakereturndata.append(","); - } - - } - if (snowflakedata.size() == 1 || i == snowflakedata.size() - 1) - snowflakereturndata.append("]"); - else - snowflakereturndata.append("],"); - - } - - snowflakereturndata.append("]}"); - - snowflakereturnstring = new String(snowflakereturndata); - - } catch (Exception e) { - // return "check exception"; - } - finally{ - if(session!=null) { - session.closeSession(); - } - } - - response.getWriter().write(snowflakereturnstring); - - } -} \ No newline at end of file diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharEncryptBulkFPE.java b/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharEncryptBulkFPE.java deleted file mode 100644 index f8398982..00000000 --- a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharEncryptBulkFPE.java +++ /dev/null @@ -1,219 +0,0 @@ -package com.example; - -import javax.crypto.Cipher; -import com.google.cloud.functions.HttpFunction; -import com.google.cloud.functions.HttpRequest; -import com.google.cloud.functions.HttpResponse; -import com.ingrian.security.nae.IngrianProvider; -import com.ingrian.security.nae.AbstractNAECipher; -import com.ingrian.security.nae.FPEParameterAndFormatSpec; -import com.ingrian.security.nae.NAEKey; -import com.ingrian.security.nae.NAESession; -import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder; -import com.ingrian.security.nae.IngrianProvider.Builder; -import com.ingrian.security.nae.NAECipher; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonPrimitive; - -import java.security.spec.AlgorithmParameterSpec; -import java.util.HashMap; -import java.util.Map; -import java.util.logging.Logger; -/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application - * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the - * data so applications or business intelligence tools do not have to change in order to use these columns. -* -* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested -* for all possible data sizes and combinations of encryption algorithms and IV, etc. -* Was tested with CM 2.11 & CADP 8.13 -* For more information on CADP see link below. -https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html -* For more information on Snowflake External Functions see link below. -https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp - -Notes: This example uses the CADP bulk API. -Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data -element has Unicode characters then the supported size limit would be 1750 characters - -Size of spec array should be same as the number of elements if user wants to use separate spec values -for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data -index. - * - *@author mwarner - * - */ -public class ThalesGCPSnowCADPCharEncryptBulkFPE implements HttpFunction { -// @Override - private static byte[][] data; - private static AlgorithmParameterSpec[] spec; - private static Integer[] snowrownbrs; - - private static int BATCHLIMIT = 10000; - private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPCharEncryptBulkFPE.class.getName()); - private static final Gson gson = new Gson(); - - public void service(HttpRequest request, HttpResponse response) throws Exception { - - String tweakAlgo = null; - String tweakData = null; - String snowflakereturnstring = null; - JsonArray snowflakedata = null; - NAESession session = null; - - - Map encryptedErrorMapTotal = new HashMap(); - try { - - String keyName = "testfaas"; - String userName = System.getenv("CMUSER"); - String password = System.getenv("CMPWD"); - int batchsize = Integer.parseInt(System.getenv("BATCHSIZE")); - if (batchsize >= BATCHLIMIT) - batchsize = BATCHLIMIT; - spec = new FPEParameterAndFormatSpec[batchsize]; - data = new byte[batchsize][]; - snowrownbrs = new Integer[batchsize]; - - try { - JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class); - JsonObject requestJson = null; - - if (requestParsed != null && requestParsed.isJsonObject()) { - requestJson = requestParsed.getAsJsonObject(); - } - - if (requestJson != null && requestJson.has("data")) { - snowflakedata = requestJson.getAsJsonArray("data"); - - } - - } catch (JsonParseException e) { - logger.severe("Error parsing JSON: " + e.getMessage()); - } - - // System.setProperty("com.ingrian.security.nae.IngrianNAE_Properties_Conf_Filename", - // "IngrianNAE.properties"); - System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", - "CADP_for_JAVA.properties"); - IngrianProvider builder = new Builder().addConfigFileInputStream( - getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build(); - // IngrianProvider builder = new - // Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build(); - session = NAESession.getSession(userName, password.toCharArray()); - NAEKey key = NAEKey.getSecretKey(keyName, session); - int row_number = 0; - - StringBuffer snowflakereturndata = new StringBuffer(); - // Serialization - snowflakereturndata.append("{ \"data\": ["); - String algorithm = "FPE/FF1/CARD62"; - - AbstractNAECipher encryptCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider"); - int i = 0; - - int totalRowsLeft = snowflakedata.size(); - while (i < snowflakedata.size()) { - int index = 0; - int dataIndex = 0; - int specIndex = 0; - int snowRowIndex = 0; - // System.out.println("totalRowsLeft begining =" + totalRowsLeft); - if (totalRowsLeft < batchsize) { - spec = new FPEParameterAndFormatSpec[totalRowsLeft]; - data = new byte[totalRowsLeft][]; - snowrownbrs = new Integer[totalRowsLeft]; - } else { - spec = new FPEParameterAndFormatSpec[batchsize]; - data = new byte[batchsize][]; - snowrownbrs = new Integer[batchsize]; - } - - for (int b = 0; b < batchsize && b < totalRowsLeft; b++) { - - JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); - - for (int j = 0; j < snowflakerow.size(); j++) { - - JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); - // System.out.print(snowflakecolumn + " "); - String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); - // get a cipher - if (j == 1) { - - // FPE example - data[dataIndex++] = sensitive.getBytes(); - spec[specIndex++] = new FPEParameterAndFormatBuilder(tweakData) - .set_tweakAlgorithm(tweakAlgo).build(); - } else { - - row_number = snowflakecolumn.getAsInt(); - snowrownbrs[snowRowIndex++] = row_number; - - } - - } - - i++; - } - // make bulk call.... - // initializing the cipher for encrypt operation - encryptCipher.init(Cipher.ENCRYPT_MODE, key, spec[0]); - - // Map to store exceptions while encryption - Map encryptedErrorMap = new HashMap(); - - // performing bulk operation - byte[][] encryptedData = encryptCipher.doFinalBulk(data, spec, encryptedErrorMap); - - for (Map.Entry entry : encryptedErrorMap.entrySet()) { - Integer mkey = entry.getKey(); - String mvalue = entry.getValue(); - encryptedErrorMapTotal.put(mkey, mvalue); - } - - for (int enc = 0; enc < encryptedData.length; enc++) { - - snowflakereturndata.append("["); - snowflakereturndata.append(snowrownbrs[enc]); - snowflakereturndata.append(","); - snowflakereturndata.append(new String(encryptedData[enc])); - - if (index <= batchsize - 1 || index < totalRowsLeft - 1) - // if (index < batchsize -1 && index < totalRowsLeft -1 ) - { - if (totalRowsLeft - 1 > 0) - snowflakereturndata.append("],"); - else - snowflakereturndata.append("]"); - } else { - - snowflakereturndata.append("]"); - } - - index++; - totalRowsLeft--; - - } - } - - snowflakereturndata.append("]}"); - - snowflakereturnstring = new String(snowflakereturndata); - - } catch (Exception e) { - // return "check exception"; - } - finally{ - if(session!=null) { - session.closeSession(); - } - } - - response.getWriter().write(snowflakereturnstring); - - } -} \ No newline at end of file diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharEncryptFPE.java b/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharEncryptFPE.java deleted file mode 100644 index 9740f080..00000000 --- a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharEncryptFPE.java +++ /dev/null @@ -1,144 +0,0 @@ -package com.example; - -import javax.crypto.Cipher; -import com.google.cloud.functions.HttpFunction; -import com.google.cloud.functions.HttpRequest; -import com.google.cloud.functions.HttpResponse; -import com.ingrian.security.nae.IngrianProvider; -import com.ingrian.security.nae.FPEParameterAndFormatSpec; -import com.ingrian.security.nae.NAEKey; -import com.ingrian.security.nae.NAESession; -import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder; -import com.ingrian.security.nae.IngrianProvider.Builder; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonPrimitive; -import java.util.logging.Logger; -/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application - * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the - * data so applications or business intelligence tools do not have to change in order to use these columns. -* -* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested -* for all possible data sizes and combinations of encryption algorithms and IV, etc. -* Was tested with CM 2.11 & CADP 8.13 -* For more information on CADP see link below. -https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html -* For more information on Snowflake External Functions see link below. -https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp - * - *@author mwarner - * - */ -public class ThalesGCPSnowCADPCharEncryptFPE implements HttpFunction { -// @Override - - private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPCharEncryptFPE.class.getName()); - private static final Gson gson = new Gson(); - - public void service(HttpRequest request, HttpResponse response) throws Exception { - - String encdata = ""; - String tweakAlgo = null; - String tweakData = null; - String snowflakereturnstring = null; - NAESession session = null; - try { - - String keyName = "testfaas"; - String userName = System.getenv("CMUSER"); - String password = System.getenv("CMPWD"); - JsonArray snowflakedata = null; - - try { - JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class); - JsonObject requestJson = null; - - if (requestParsed != null && requestParsed.isJsonObject()) { - requestJson = requestParsed.getAsJsonObject(); - } - - if (requestJson != null && requestJson.has("data")) { - snowflakedata = requestJson.getAsJsonArray("data"); - - } - - } catch (JsonParseException e) { - logger.severe("Error parsing JSON: " + e.getMessage()); - } - - // System.setProperty("com.ingrian.security.nae.IngrianNAE_Properties_Conf_Filename", - // "IngrianNAE.properties"); - System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", - "CADP_for_JAVA.properties"); - IngrianProvider builder = new Builder().addConfigFileInputStream( - getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build(); - // IngrianProvider builder = new - // Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build(); - session = NAESession.getSession(userName, password.toCharArray()); - NAEKey key = NAEKey.getSecretKey(keyName, session); - int row_number = 0; - - StringBuffer snowflakereturndata = new StringBuffer(); - // Serialization - snowflakereturndata.append("{ \"data\": ["); - String algorithm = "FPE/FF1/CARD62"; - // String algorithm = "AES/CBC/PKCS5Padding"; - FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo) - .build(); - - Cipher encryptCipher = Cipher.getInstance(algorithm, "IngrianProvider"); - - for (int i = 0; i < snowflakedata.size(); i++) { - JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); - snowflakereturndata.append("["); - for (int j = 0; j < snowflakerow.size(); j++) { - - JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); - String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); - // get a cipher - if (j == 1) { - - // FPE example - - // initialize cipher to encrypt. - encryptCipher.init(Cipher.ENCRYPT_MODE, key, param); - // encrypt data - byte[] outbuf = encryptCipher.doFinal(sensitive.getBytes()); - encdata = new String(outbuf); - - snowflakereturndata.append(encdata); - - } else { - row_number = snowflakecolumn.getAsInt(); - snowflakereturndata.append(row_number); - snowflakereturndata.append(","); - } - - } - if (snowflakedata.size() == 1 || i == snowflakedata.size() - 1) - snowflakereturndata.append("]"); - else - snowflakereturndata.append("],"); - - } - - snowflakereturndata.append("]}"); - - snowflakereturnstring = new String(snowflakereturndata); - - } catch (Exception e) { - // return "check exception"; - } - finally{ - if(session!=null) { - session.closeSession(); - } - } - - response.getWriter().write(snowflakereturnstring); - - } -} \ No newline at end of file diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrDecryptBulkFPE.java b/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrDecryptBulkFPE.java deleted file mode 100644 index 55f24572..00000000 --- a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrDecryptBulkFPE.java +++ /dev/null @@ -1,218 +0,0 @@ -package com.example; - -import javax.crypto.Cipher; -import com.google.cloud.functions.HttpFunction; -import com.google.cloud.functions.HttpRequest; -import com.google.cloud.functions.HttpResponse; -import com.ingrian.security.nae.IngrianProvider; -import com.ingrian.security.nae.AbstractNAECipher; -import com.ingrian.security.nae.FPEParameterAndFormatSpec; -import com.ingrian.security.nae.NAEKey; -import com.ingrian.security.nae.NAESession; -import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder; -import com.ingrian.security.nae.IngrianProvider.Builder; -import com.ingrian.security.nae.NAECipher; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonPrimitive; - -import java.security.spec.AlgorithmParameterSpec; -import java.util.HashMap; -import java.util.Map; -import java.util.logging.Logger; - -/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application - * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the - * data so applications or business intelligence tools do not have to change in order to use these columns. -* -* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested -* for all possible data sizes and combinations of encryption algorithms and IV, etc. -* Was tested with CM 2.11 & CADP 8.13 -* For more information on CADP see link below. -https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html -* For more information on Snowflake External Functions see link below. -https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp - -Notes: This example uses the CADP bulk API. -Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data -element has Unicode characters then the supported size limit would be 1750 characters - -Size of spec array should be same as the number of elements if user wants to use separate spec values -for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data -index. - * - *@author mwarner - * - */ -public class ThalesGCPSnowCADPNbrDecryptBulkFPE implements HttpFunction { -// @Override - private static byte[][] data; - private static AlgorithmParameterSpec[] spec; - private static Integer[] snowrownbrs; - - private static int BATCHLIMIT = 10000; - private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPNbrDecryptBulkFPE.class.getName()); - private static final Gson gson = new Gson(); - - public void service(HttpRequest request, HttpResponse response) throws Exception { - - String tweakAlgo = null; - String tweakData = null; - String snowflakereturnstring = null; - JsonArray snowflakedata = null; - NAESession session = null; - Map encryptedErrorMapTotal = new HashMap(); - try { - - String keyName = "testfaas"; - String userName = System.getenv("CMUSER"); - String password = System.getenv("CMPWD"); - int batchsize = Integer.parseInt(System.getenv("BATCHSIZE")); ; - if (batchsize >= BATCHLIMIT) - batchsize = BATCHLIMIT; - spec = new FPEParameterAndFormatSpec[batchsize]; - data = new byte[batchsize][]; - snowrownbrs = new Integer[batchsize]; - - try { - JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class); - JsonObject requestJson = null; - - if (requestParsed != null && requestParsed.isJsonObject()) { - requestJson = requestParsed.getAsJsonObject(); - } - - if (requestJson != null && requestJson.has("data")) { - snowflakedata = requestJson.getAsJsonArray("data"); - - } - - } catch (JsonParseException e) { - logger.severe("Error parsing JSON: " + e.getMessage()); - } - - // System.setProperty("com.ingrian.security.nae.IngrianNAE_Properties_Conf_Filename", - // "IngrianNAE.properties"); - System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", - "CADP_for_JAVA.properties"); - IngrianProvider builder = new Builder().addConfigFileInputStream( - getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build(); - // IngrianProvider builder = new - // Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build(); - session = NAESession.getSession(userName, password.toCharArray()); - NAEKey key = NAEKey.getSecretKey(keyName, session); - int row_number = 0; - - StringBuffer snowflakereturndata = new StringBuffer(); - // Serialization - snowflakereturndata.append("{ \"data\": ["); - String algorithm = "FPE/FF1/CARD10"; - - AbstractNAECipher decryptCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider"); - int i = 0; - - int totalRowsLeft = snowflakedata.size(); - while (i < snowflakedata.size()) { - int index = 0; - int dataIndex = 0; - int specIndex = 0; - int snowRowIndex = 0; - // System.out.println("totalRowsLeft begining =" + totalRowsLeft); - if (totalRowsLeft < batchsize) { - spec = new FPEParameterAndFormatSpec[totalRowsLeft]; - data = new byte[totalRowsLeft][]; - snowrownbrs = new Integer[totalRowsLeft]; - } else { - spec = new FPEParameterAndFormatSpec[batchsize]; - data = new byte[batchsize][]; - snowrownbrs = new Integer[batchsize]; - } - - for (int b = 0; b < batchsize && b < totalRowsLeft; b++) { - - JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); - - for (int j = 0; j < snowflakerow.size(); j++) { - - JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); - // System.out.print(snowflakecolumn + " "); - String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); - // get a cipher - if (j == 1) { - - // FPE example - data[dataIndex++] = sensitive.getBytes(); - spec[specIndex++] = new FPEParameterAndFormatBuilder(tweakData) - .set_tweakAlgorithm(tweakAlgo).build(); - } else { - - row_number = snowflakecolumn.getAsInt(); - snowrownbrs[snowRowIndex++] = row_number; - - } - - } - - i++; - } - // make bulk call.... - // initializing the cipher for encrypt operation - decryptCipher.init(Cipher.DECRYPT_MODE, key, spec[0]); - - // Map to store exceptions while encryption - Map decryptErrorMap = new HashMap(); - - // performing bulk operation - byte[][] decryptedData = decryptCipher.doFinalBulk(data, spec, decryptErrorMap); - - for (Map.Entry entry : decryptErrorMap.entrySet()) { - Integer mkey = entry.getKey(); - String mvalue = entry.getValue(); - encryptedErrorMapTotal.put(mkey, mvalue); - } - - for (int enc = 0; enc < decryptedData.length; enc++) { - - snowflakereturndata.append("["); - snowflakereturndata.append(snowrownbrs[enc]); - snowflakereturndata.append(","); - snowflakereturndata.append(new String(decryptedData[enc])); - - if (index <= batchsize - 1 || index < totalRowsLeft - 1) - // if (index < batchsize -1 && index < totalRowsLeft -1 ) - { - if (totalRowsLeft - 1 > 0) - snowflakereturndata.append("],"); - else - snowflakereturndata.append("]"); - } else { - - snowflakereturndata.append("]"); - } - - index++; - totalRowsLeft--; - - } - } - - snowflakereturndata.append("]}"); - - snowflakereturnstring = new String(snowflakereturndata); - - } catch (Exception e) { - // return "check exception"; - } - finally{ - if(session!=null) { - session.closeSession(); - } - } - - response.getWriter().write(snowflakereturnstring); - - } -} \ No newline at end of file diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrDecryptFPE.java b/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrDecryptFPE.java deleted file mode 100644 index c67ef8c6..00000000 --- a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrDecryptFPE.java +++ /dev/null @@ -1,146 +0,0 @@ -package com.example; - - -import javax.crypto.Cipher; -import com.google.cloud.functions.HttpFunction; -import com.google.cloud.functions.HttpRequest; -import com.google.cloud.functions.HttpResponse; -import com.ingrian.security.nae.IngrianProvider; -import com.ingrian.security.nae.FPEParameterAndFormatSpec; -import com.ingrian.security.nae.NAEKey; -import com.ingrian.security.nae.NAESession; -import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder; -import com.ingrian.security.nae.IngrianProvider.Builder; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonPrimitive; -import java.util.logging.Logger; - -/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application - * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the - * data so applications or business intelligence tools do not have to change in order to use these columns. -* -* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested -* for all possible data sizes and combinations of encryption algorithms and IV, etc. -* Was tested with CM 2.11 & CADP 8.13 -* For more information on CADP see link below. -https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html -* For more information on Snowflake External Functions see link below. -https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp - * - *@author mwarner - * - */ -public class ThalesGCPSnowCADPNbrDecryptFPE implements HttpFunction { -// @Override - - private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPNbrDecryptFPE.class.getName()); - private static final Gson gson = new Gson(); - public void service(HttpRequest request, HttpResponse response) - throws Exception { - - String decData = ""; - String tweakAlgo = null; - String tweakData = null; - String snowflakereturnstring = null; - NAESession session = null; - - try { - - String keyName = "testfaas"; - String userName = System.getenv("CMUSER"); - String password = System.getenv("CMPWD"); - JsonArray snowflakedata = null; - - try { - JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class); - JsonObject requestJson = null; - - if (requestParsed != null && requestParsed.isJsonObject()) { - requestJson = requestParsed.getAsJsonObject(); - } - - if (requestJson != null && requestJson.has("data")) { - snowflakedata = requestJson.getAsJsonArray("data"); - - } - - } catch (JsonParseException e) { - logger.severe("Error parsing JSON: " + e.getMessage()); - } - - // System.setProperty("com.ingrian.security.nae.IngrianNAE_Properties_Conf_Filename", "IngrianNAE.properties"); - System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", "CADP_for_JAVA.properties"); - IngrianProvider builder = new Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build(); - //IngrianProvider builder = new Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build(); - session = NAESession.getSession(userName, password.toCharArray()); - NAEKey key = NAEKey.getSecretKey(keyName, session); - - int row_number = 0; - - StringBuffer snowflakereturndata = new StringBuffer(); - // Serialization - snowflakereturndata.append("{ \"data\": ["); - String algorithm = "FPE/FF1/CARD10"; - // String algorithm = "AES/CBC/PKCS5Padding"; - FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo) - .build(); - - Cipher decryptCipher = Cipher.getInstance(algorithm, "IngrianProvider"); - - for (int i = 0; i < snowflakedata.size(); i++) { - JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); - snowflakereturndata.append("["); - for (int j = 0; j < snowflakerow.size(); j++) { - - JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); - - String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); - // get a cipher - if (j == 1) { - - // FPE example - - // initialize cipher to encrypt. - decryptCipher.init(Cipher.DECRYPT_MODE, key, param); - // encrypt data - byte[] outbuf = decryptCipher.doFinal(sensitive.getBytes()); - decData = new String(outbuf); - //System.out.println("Enc data : " + encdata); - - snowflakereturndata.append(decData); - - } else { - row_number = snowflakecolumn.getAsInt(); - snowflakereturndata.append(row_number); - snowflakereturndata.append(","); - } - - } - if ( snowflakedata.size() == 1 || i == snowflakedata.size() -1) - snowflakereturndata.append("]"); - else - snowflakereturndata.append("],"); - - } - - snowflakereturndata.append("]}"); - - snowflakereturnstring = new String(snowflakereturndata); - - } catch (Exception e) { - // return "check exception"; - } - finally{ - if(session!=null) { - session.closeSession(); - } - } - - response.getWriter().write(snowflakereturnstring); - - } -} \ No newline at end of file diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrEncryptBulkFPE.java b/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrEncryptBulkFPE.java deleted file mode 100644 index 2d68d011..00000000 --- a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrEncryptBulkFPE.java +++ /dev/null @@ -1,220 +0,0 @@ -package com.example; - -import javax.crypto.Cipher; -import com.google.cloud.functions.HttpFunction; -import com.google.cloud.functions.HttpRequest; -import com.google.cloud.functions.HttpResponse; -import com.ingrian.security.nae.IngrianProvider; -import com.ingrian.security.nae.AbstractNAECipher; -import com.ingrian.security.nae.FPEParameterAndFormatSpec; -import com.ingrian.security.nae.NAEKey; -import com.ingrian.security.nae.NAESession; -import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder; -import com.ingrian.security.nae.IngrianProvider.Builder; -import com.ingrian.security.nae.NAECipher; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonPrimitive; - -import java.security.spec.AlgorithmParameterSpec; -import java.util.HashMap; -import java.util.Map; -import java.util.logging.Logger; - -/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application - * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the - * data so applications or business intelligence tools do not have to change in order to use these columns. -* -* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested -* for all possible data sizes and combinations of encryption algorithms and IV, etc. -* Was tested with CM 2.11 & CADP 8.13 -* For more information on CADP see link below. -https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html -* For more information on Snowflake External Functions see link below. -https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp - -Notes: This example uses the CADP bulk API. -Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data -element has Unicode characters then the supported size limit would be 1750 characters - -Size of spec array should be same as the number of elements if user wants to use separate spec values -for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data -index. - * - *@author mwarner - * - */ -public class ThalesGCPSnowCADPNbrEncryptBulkFPE implements HttpFunction { -// @Override - private static byte[][] data; - private static AlgorithmParameterSpec[] spec; - private static Integer[] snowrownbrs; - - private static int BATCHLIMIT = 10000; - private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPNbrEncryptBulkFPE.class.getName()); - private static final Gson gson = new Gson(); - - public void service(HttpRequest request, HttpResponse response) throws Exception { - - String tweakAlgo = null; - String tweakData = null; - String snowflakereturnstring = null; - JsonArray snowflakedata = null; - Map encryptedErrorMapTotal = new HashMap(); - - NAESession session = null; - try { - - String keyName = "testfaas"; - String userName = System.getenv("CMUSER"); - String password = System.getenv("CMPWD"); - int batchsize = Integer.parseInt(System.getenv("BATCHSIZE")); - if (batchsize >= BATCHLIMIT) - batchsize = BATCHLIMIT; - spec = new FPEParameterAndFormatSpec[batchsize]; - data = new byte[batchsize][]; - snowrownbrs = new Integer[batchsize]; - - try { - JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class); - JsonObject requestJson = null; - - if (requestParsed != null && requestParsed.isJsonObject()) { - requestJson = requestParsed.getAsJsonObject(); - } - - if (requestJson != null && requestJson.has("data")) { - snowflakedata = requestJson.getAsJsonArray("data"); - - } - - } catch (JsonParseException e) { - logger.severe("Error parsing JSON: " + e.getMessage()); - } - - // System.setProperty("com.ingrian.security.nae.IngrianNAE_Properties_Conf_Filename", - // "IngrianNAE.properties"); - System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", - "CADP_for_JAVA.properties"); - IngrianProvider builder = new Builder().addConfigFileInputStream( - getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build(); - // IngrianProvider builder = new - // Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build(); - session = NAESession.getSession(userName, password.toCharArray()); - NAEKey key = NAEKey.getSecretKey(keyName, session); - int row_number = 0; - - StringBuffer snowflakereturndata = new StringBuffer(); - // Serialization - snowflakereturndata.append("{ \"data\": ["); - String algorithm = "FPE/FF1/CARD10"; - - AbstractNAECipher encryptCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider"); - int i = 0; - - int totalRowsLeft = snowflakedata.size(); - while (i < snowflakedata.size()) { - int index = 0; - int dataIndex = 0; - int specIndex = 0; - int snowRowIndex = 0; - // System.out.println("totalRowsLeft begining =" + totalRowsLeft); - if (totalRowsLeft < batchsize) { - spec = new FPEParameterAndFormatSpec[totalRowsLeft]; - data = new byte[totalRowsLeft][]; - snowrownbrs = new Integer[totalRowsLeft]; - } else { - spec = new FPEParameterAndFormatSpec[batchsize]; - data = new byte[batchsize][]; - snowrownbrs = new Integer[batchsize]; - } - - for (int b = 0; b < batchsize && b < totalRowsLeft; b++) { - - JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); - - for (int j = 0; j < snowflakerow.size(); j++) { - - JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); - // System.out.print(snowflakecolumn + " "); - String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); - // get a cipher - if (j == 1) { - - // FPE example - data[dataIndex++] = sensitive.getBytes(); - spec[specIndex++] = new FPEParameterAndFormatBuilder(tweakData) - .set_tweakAlgorithm(tweakAlgo).build(); - } else { - - row_number = snowflakecolumn.getAsInt(); - snowrownbrs[snowRowIndex++] = row_number; - - } - - } - - i++; - } - // make bulk call.... - // initializing the cipher for encrypt operation - encryptCipher.init(Cipher.ENCRYPT_MODE, key, spec[0]); - - // Map to store exceptions while encryption - Map encryptedErrorMap = new HashMap(); - - // performing bulk operation - byte[][] encryptedData = encryptCipher.doFinalBulk(data, spec, encryptedErrorMap); - - for (Map.Entry entry : encryptedErrorMap.entrySet()) { - Integer mkey = entry.getKey(); - String mvalue = entry.getValue(); - encryptedErrorMapTotal.put(mkey, mvalue); - } - - for (int enc = 0; enc < encryptedData.length; enc++) { - - snowflakereturndata.append("["); - snowflakereturndata.append(snowrownbrs[enc]); - snowflakereturndata.append(","); - snowflakereturndata.append(new String(encryptedData[enc])); - - if (index <= batchsize - 1 || index < totalRowsLeft - 1) - // if (index < batchsize -1 && index < totalRowsLeft -1 ) - { - if (totalRowsLeft - 1 > 0) - snowflakereturndata.append("],"); - else - snowflakereturndata.append("]"); - } else { - - snowflakereturndata.append("]"); - } - - index++; - totalRowsLeft--; - - } - } - - snowflakereturndata.append("]}"); - - snowflakereturnstring = new String(snowflakereturndata); - - - } catch (Exception e) { - // return "check exception"; - } - finally{ - if(session!=null) { - session.closeSession(); - } - } - - response.getWriter().write(snowflakereturnstring); - - } -} \ No newline at end of file diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrEncryptFPE.java b/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrEncryptFPE.java deleted file mode 100644 index 03357e0d..00000000 --- a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrEncryptFPE.java +++ /dev/null @@ -1,144 +0,0 @@ -package com.example; - - -import javax.crypto.Cipher; -import com.google.cloud.functions.HttpFunction; -import com.google.cloud.functions.HttpRequest; -import com.google.cloud.functions.HttpResponse; -import com.ingrian.security.nae.IngrianProvider; -import com.ingrian.security.nae.FPEParameterAndFormatSpec; -import com.ingrian.security.nae.NAEKey; -import com.ingrian.security.nae.NAESession; -import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder; -import com.ingrian.security.nae.IngrianProvider.Builder; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonPrimitive; -import java.util.logging.Logger; - -/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application - * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the - * data so applications or business intelligence tools do not have to change in order to use these columns. -* -* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested -* for all possible data sizes and combinations of encryption algorithms and IV, etc. -* Was tested with CM 2.11 & CADP 8.13 -* For more information on CADP see link below. -https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html -* For more information on Snowflake External Functions see link below. -https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp - * - *@author mwarner - * - */ -public class ThalesGCPSnowCADPNbrEncryptFPE implements HttpFunction { -// @Override - - private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPNbrEncryptFPE.class.getName()); - private static final Gson gson = new Gson(); - public void service(HttpRequest request, HttpResponse response) - throws Exception { - - String encdata = ""; - String tweakAlgo = null; - String tweakData = null; - String snowflakereturnstring = null; - NAESession session = null; - try { - - String keyName = "testfaas"; - String userName = System.getenv("CMUSER"); - String password = System.getenv("CMPWD"); - JsonArray snowflakedata = null; - - try { - JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class); - JsonObject requestJson = null; - - if (requestParsed != null && requestParsed.isJsonObject()) { - requestJson = requestParsed.getAsJsonObject(); - } - - if (requestJson != null && requestJson.has("data")) { - snowflakedata = requestJson.getAsJsonArray("data"); - - } - - } catch (JsonParseException e) { - logger.severe("Error parsing JSON: " + e.getMessage()); - } - - - System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", "CADP_for_JAVA.properties"); - IngrianProvider builder = new Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build(); - //IngrianProvider builder = new Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build(); - session = NAESession.getSession(userName, password.toCharArray()); - NAEKey key = NAEKey.getSecretKey(keyName, session); - int row_number = 0; - - StringBuffer snowflakereturndata = new StringBuffer(); - // Serialization - snowflakereturndata.append("{ \"data\": ["); - String algorithm = "FPE/FF1/CARD10"; - // String algorithm = "AES/CBC/PKCS5Padding"; - FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo) - .build(); - - Cipher encryptCipher = Cipher.getInstance(algorithm, "IngrianProvider"); - - for (int i = 0; i < snowflakedata.size(); i++) { - JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); - snowflakereturndata.append("["); - for (int j = 0; j < snowflakerow.size(); j++) { - - JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); - - String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); - // get a cipher - if (j == 1) { - - // FPE example - - // initialize cipher to encrypt. - encryptCipher.init(Cipher.ENCRYPT_MODE, key, param); - // encrypt data - byte[] outbuf = encryptCipher.doFinal(sensitive.getBytes()); - encdata = new String(outbuf); - - snowflakereturndata.append(encdata); - - } else { - row_number = snowflakecolumn.getAsInt(); - snowflakereturndata.append(row_number); - snowflakereturndata.append(","); - } - - } - if ( snowflakedata.size() == 1 || i == snowflakedata.size() -1) - snowflakereturndata.append("]"); - else - snowflakereturndata.append("],"); - - } - - snowflakereturndata.append("]}"); - - snowflakereturnstring = new String(snowflakereturndata); - - - } catch (Exception e) { - // return "check exception"; - } - finally{ - if(session!=null) { - session.closeSession(); - } - } - - response.getWriter().write(snowflakereturnstring); - - } -} \ No newline at end of file diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/resources/CADP_for_JAVA.properties b/database/snowflake/CADP-SNOW-GCP-Functions/src/main/resources/CADP_for_JAVA.properties deleted file mode 100644 index 2814776e..00000000 --- a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/resources/CADP_for_JAVA.properties +++ /dev/null @@ -1,657 +0,0 @@ -# -# Copyright (c) 2002-2020 SafeNet, Inc. -# -# -# Ingrian Network-Attached Encryption (NAE) properties file -# -# Release Version: 8.12.0.000-000 -# - -#[Version] -# Version of the properties file for the Ingrian JCE provider. -# -# Do not modify this property. -# -Version=2.4 - - -#[Network Configuration] -# [NAE Server IP and Port] -# The IP address and port of the NAE server. -# -# Multiple IPs can be specified when load balancing is used. -# For all tier-aware parameters, the tier is indicated with a trailing -# :n after the parameter name, i.e. NAE_IP.1=127.0.0.1 -# Setting the parameter with no tier sets the default value for all tiers. -# i.e. Connection_Timeout=600000 sets Connection_Timeout for all tiers while -# Connection_Timeout.1=700000 sets Connection_Timeout for tier 1. -# -# Multiple IPs are separated by colons, e.g., -# 192.168.1.10:192.168.1.11:192.168.1.12 -# For IPv6, the IP address is to be specified in curly braces, such as {2001:0db8:85a3:0000:0000:8a2e:0370:7334}. Also, combination of IPv4 and IPv6 IP -# addresses can be used separated by colons(:) provided each IPv6 IP address is within {}. The IPv6 is supported for the -# KMIP interfaces too. -# {IPv6}:IPv4:{IPv6} -# IPv4:{IPv6} -# IPv4:{IPv6}:IPv4 -# {IPv6}:IPv4:IPv4:{IPv6} -#NAE_IP.1=yourip -NAE_IP.1=yourip - -#[Network Configuration] -# [NAE Server Port] -# NAE_Port is tier-aware -# Do not set the port value to 9443 because this is the port typically used -# to connect to the management console. -NAE_Port=9001 - -#[Network Configuration] -# [Ignore_DNS_Resolution_Failure] -# Ignore_DNS_Resolution_Failure is used in case either JCE Application fails to locate DNS Server. -# or If Host Name present in any Ip Field is not getting resolved through DNS.Application will shift to Persistance chache if it is on. -Ignore_DNS_Resolution_Failure=false - -#[Network Configuration] -# [KMIP Server Port] -# KMIP_Port is tier-aware -# By convention, the KMIP standard port is 5696, but this is not enforced, -# or taken as default by JCE. Must be a KMIP SSL port on the server. -# Do not set the port value to 9443 because this is the port typically used -# to connect to the management console. -KMIP_Port=5696 - -#[Network Configuration] -# [Protocol] -# The protocol used between the client and the NAE server. -# -# Protocol is tier-aware. -# Valid values: tcp, ssl. -# Default: tcp -# Recommended: ssl -# -Protocol=tcp - -# All connection configuration properties effect the connection set up. -# -#[Connection Configuration] -# [Verify_SSL_Certificate] -# This property is considered only when Protocol is set to ssl. -# -# Enable or disable verification of DataSecure IP address/host name -# against Subject Common Name (CN) or Subject Alternative Name (DNS or IP) -# in the certificate. -# -# Valid values: yes, no. -# Default: no -# Recommended: yes -# -Verify_SSL_Certificate=no - -#[Connection Configuration] -# [SSL_Handshake_Timeout] -# This property is considered only when Protocol is set to ssl. -# -# Allocates a time for SSL handshake. If SSL handshake is not complete -# within this time period, the connection is closed. -# -# The value is specified in milliseconds. -# -# If the value is set to 0 or is not set, SSL handshake timeout -# is not enforced. -# -SSL_Handshake_Timeout= - -#[Connection Configuration] -# [Persistent Connections] -# Enable or disable persistent connections. -# -# If enabled, the client will use a pool of persistent connections to the -# NAE server. If disabled, a new connection will be created and then -# closed for each request. -# -# Valid values: yes, no. -# Default: yes -# Recommended: yes -# -Use_Persistent_Connections=yes - - -#[Connection Configuration] -# [Connection Pooling] -# The maximum number of connections in the persistent connection pool per session. -# -# This value is used only when persistent connections are enabled. -# -# Size_of_Connection_Pool is tier-aware. -# Default: 300 -# -Size_of_Connection_Pool=300 - -#[Connection Configuration] -#[Load Balancing Configuration] -# The type of load balancing. -# -# This value is only relevant if you are load balancing across multiple -# NAE servers. -# -# Valid values: random, round-robin. -# Default: round-robin -# -Load_Balancing_Algorithm=round-robin - -#[Connection Configuration] -# [Connection Idle Timeout] -# The time a connection is allowed to be idle in the connection pool -# before it gets closed automatically by the client. -# -# The timeout can be specified in any time units (default - milliseconds). The client -# will check how long each connection has been idle for. If the time has passed the value -# specified here, the client will close the connection and remove it from -# the connection pool. To be effective, this setting must be less than the -# Connection Timeout setting in the NAE Server Settings section in the -# Management Console of the NAE server. -# -# Setting this value to 0 is equivalent to an infinite timeout. -# Connection_Idle_Timeout is tier-aware. -# Default: 600000 -# -Connection_Idle_Timeout=600000 - -#[Connection Configuration] -# [Unreachable Server Retry] -# The amount of time to try establishing a connection on a load balancer with the server. -# -# The retry period can be specified in any time units (default - milliseconds). An error is returned -# after the specified period if no server in the pool becomes reachable. -# If logging is enabled, error messages will be logged to the log file. -# -# Setting this value to -1 is equivalent to an infinite retry period. The -# client will keep trying to connect to a server until a connection is -# established. -# -# Setting this value to -1 is not compatible with multi-tier load -# balancing because the load balancer will never switch to the secondary -# or tertiary pools. -# -# Setting the value 0 means no retrying on the particular load balancer in case all the server -# are down. It will move to next load balancer if available. -# -# Default: 60000 (1 minute) -# -Unreachable_Server_Retry_Period=60000 - -#[Connection Configuration] -# [Maximum_Server_Retry_Period] -# The total amount of time to spend trying to make connections on any tier. -# This value only has meaning when using multi-tiered load balancing. -# If this value is set to -1 (try forever), the connection manager will try -# every server on every tier continually, until one answers. -# If this value is enabled, the connection manager will try to make connections -# for at least Maximum_Server_Retry_Period time but will return -# an error if no connection can be made on the tier in use when -# Maximum_Server_Retry_Period expires. -# Server tries to make connection on each tier serially. So, new connection will -# be created whether other connection mark any tier disable or not. -# Maximum_Server_Retry_Period is tier-aware. -# -# Default: 0 (disabled) -# -Maximum_Server_Retry_Period=0 - -#[Connection Configuration] -# [Connection Timeout] -# The timeout when connecting to the NAE server. -# -# The timeout can be specified in any time units (default - milliseconds). The client -# will wait for the specified time when trying to connect to each NAE -# server. If the timeout expires, an NAEException will be thrown. -# -# Setting this value to 0 is equivalent to an infinite timeout. -# -# Caution: Setting this value too low (but not 0) may cause connections to -# fail when the NAE servers and/or network are under load. -# -# Connection_Timeout is tier-aware. -# Default: 1 minute -# -Connection_Timeout=1m - - -#[Connection Configuration] -# [Connection Read Timeout] -# The timeout when reading from the NAE server. -# -# The timeout can be specified in any time units (default - milliseconds). The client -# will wait for the specified time when trying to read data from the NAE -# server. If the timeout expires, an NAEException will be thrown. -# -# Setting this value to 0 is equivalent to an infinite timeout. -# Connection_Read_Timeout is tier-aware. -# Default: 7000 -# -Connection_Read_Timeout=7000 - -#[Connection Configuration] -# [Connection Retry] -# The amount of time to wait before trying to reconnect to a disabled -# server. -# -# The retry interval can be specified in any time units (default - milliseconds). -# If one of the NAE servers in a load balanced configuration is not reachable, -# the client will disable this server, and then wait for the specified -# time before trying to connect to it again. -# -# Setting this value to 0 is equivalent to an infinite retry interval -# (meaning the disabled server will never be brought back into use). -# -# Connection_Retry_Interval is tier-aware. -# -# Default: 600000 -# -Connection_Retry_Interval=600000 - -#[Client Certificate Configuration] -# [Client Certificate Alias] -# The client certificate to present to the NAE server. -# -# This value is only relevant when client certificate authentication is -# enabled on the NAE server. The certificate must be in PEM format. -# -# When there are multiple client certificates in your keystore, you can -# specify which certificate gets sent to the NAE server. If you do not -# specify an alias, the first certificate in the keystore will be sent. -# -# You should provide the alias of the client certificate. -# Client_Cert_Alias is tier-aware. -# No default. -# -Client_Cert_Alias= - - -#[Client Certificate Configuration] -# [Client Private Key Passphrase] -# The passphrase to unlock the client private key. -# Client_Cert_Passphrase is tier-aware. -# No default. -# -Client_Cert_Passphrase= -#[Etoken Configuration] -# Enable or disable Etoken. -# -# If enabled, the client will use a Etoken to make ssl connection with -# DS server. If disabled, a new connection will be created using local -# keystore for SSL Connection. This configuration will work only when -# protocol is ssl. -# -# Valid values: yes, no. -# Default: no -Use_Etoken=no -#[Etoken Configuration] -# The Etoken Name is used to connect with -# PKCS#11 hardware. -# -Etoken_Name= - -#[SSL/TLS Configuration] -#[Certificate Configuration] -# Location of the key store where user stores certificates (both CA certificates and end user certificates) -# and private keys. -# Key_Store_Location is tier-aware. -# -# Or Location of Etoken library in case on Use_Etoken is enabled. Etoken Library is required to load -# Etoken Keystore. -# No default. ; -Key_Store_Location=C:\\code\\aws-lambda\\certs\\cacerts - -#[SSL/TLS Configuration] -#[Certificate Configuration] -# Password to unlock keystore -# Key_Store_Password is tier-aware. -# Or -# Etoken Pin is case of Use_Etoken is enabled. -# No default. -Key_Store_Password=yourpwd - -#[Connection Configuration] -# [Cluster_Synchronization_Delay] -# The total amount of time to spend trying to make requests on keys -# go to the same device the key create or latest key modify went to. -# -# A device tries to replicate key information to other devices in the -# cluster after it completes a key create or modify request. Until -# that replication completes, requests on the key need to go to the -# device pushing the replication. -# -# If replication fails, the device waits for 40 seconds, then -# tries again. If four replications fail, the device stops trying -# to replicate data. -# -# The default is 170 seconds: 4 times 40 seconds plus a few extra -# seconds per try for network latency. For larger clusters additional -# time may be needed. -# -# Disable the function: 0 seconds -# -# Default: 170 seconds -# -Cluster_Synchronization_Delay=170 - -# [Client Key Caching] -# [Symmetric_Key_Cache_Enabled] -# Enables symmetric key caching. -# -# If enabled, the client will be able to use symmetric keys to encrypt data -# locally. If disabled, only remote encryption will be supported. Should only be -# enabled with Protocol set to ssl. To allow key caching over unsecured -# communication, set this variable to tcp_ok -# -# Valid values: yes, no, tcp_ok -# Default: no -# Recommended: no -Symmetric_Key_Cache_Enabled=tcp_ok -Asymmetric_Key_Cache_Enabled=tcp_ok - -# [Client Key Caching] -# [Symmetric_Key_Cache_Expiry] -# Time period since key was cached after which a symmetric key -# may be removed from cache. Symmetric_Key_Cache_Expiry can be specified -# in any time units (default - seconds) -# Setting this value to 0 is equivalent to an infinite timeout. -# Note: This field is also applicable to Asymmetric key cache expiry -# Default: 43200 (12 hours) -Symmetric_Key_Cache_Expiry=43200 - -# [Client Key Caching] -# [Symmetric_Key_Cache_AutoRefresh_Interval] -# It is a time duration after which a cached key will become eligible for refresh in cache. -# Actual key refresh will only be initiated when the cached key is queried from cache after -# it becomes eligible for refresh before its expiry time. If Persistent Cache is enabled then Key -# will be refreshed both in Symmetric and Persistent Cache during refresh operation. -# Auto refresh cache operation will only be enabled when this interval value is greater than 0. -# Symmetric_Key_Cache_AutoRefresh_Interval can be specified in any time unit (default unit -seconds). -#default :0 - -Symmetric_Key_Cache_AutoRefresh_Interval=0 - - -# [Client Key Caching] -# [Local_Cipher_Cache_Expiry] -# Time period since local cipher was initialize with cached key. -# Local_Cipher_Expiry can be specified in any time units (default - milliseconds) -# Setting this value to 0 is equivalent to no timeout which means cipher will -# expire after every operation. -# To do local cipher expiry, Symmetric_Key_Cache_Enabled must be set to -# "yes" or "tcp_ok. -# -# Note: Local_Cipher_Expiry timeout should be less than Symmetric_Key_Cache_Expiry -# because all ciphers will get expired on symmetric cache expiry. -# Default: 0 (cipher will expiry every time) -Local_Cipher_Cache_Expiry=0 - -# [Client Key Caching] -# [Local_Crypto_Provider] -# Name of JCE provider to perform local crypto -# -# Default: SunJCE or IBMJCE (depends on JVM) -Local_Crypto_Provider= - -# [Persistent Key Caching] -# [Persistent_Cache_Enabled] -# Enables persistent key caching during local encryption. -# -# To persist symmetric keys Symmetric_Key_Cache_Enabled must be set to -# "yes" or "tcp_ok", Persistent_Cache_Enabled must be set to "yes", -# Persistent_Cache_Expiry set to a zero or positive value, and -# Persistent_Cache_Directory set to directory. -# If Symmetric_Key_Cache_Enabled or Public_Key_Cache_Enabled is set -# to "no", all Persistent_Cache_* properties will be ignored. -# -# Valid values: yes, no -# Default: no -# Recommended: no -Persistent_Cache_Enabled=no - - -# [Persistent Key Caching] -# [Persistent_Cache_Expiry_Keys] -# -# Time since stored after which a key may be removed from key cache. -# Settiing to 0 would mean no timeout and keys would not be removed -# from key cache. Persistent_Cache_Expiry_Keys can be specified -# in any time units (default - seconds) -# -# Default: 43200 (12 hours) -Persistent_Cache_Expiry_Keys=43200 - -# [Persistent Key Caching] -# [Persistent_Cache_Directory] -# Specifies the location of the persistent cache; error if not specified. -# -# Default: none -# -Persistent_Cache_Directory= - -# [Persistent Key Caching] -# [Persistent_Cache_Max_Size] -# Limits the number of keys, certificates, and crypto profiles in persistent cache -# -# Default: 100 -Persistent_Cache_Max_Size=100 - - -######################################################################### -# THE FOLLOWING PROPERTIES ONLY NEED TO BE SET IF YOU ARE INSTALLING THE -# INGRIAN JCE PROVIDER AS PART OF AN ORACLE OR DB2 DATABASE CONNECTOR; -# OTHERWISE, THEY ARE DISREGARDED. -######################################################################### - -#[SSL/TLS Configuration] -# [CA Certificate for Server Authentication] -# The CA certificate that signed the NAE server certificate presented to -# clients to establish SSL connections. -# -# If you are using SSL between the client and server, you must specify a -# path to the CA certificate that signed the NAE server certificate. If -# the client cannot validate the certificate presented by the NAE server, -# the client will not be able to establish an SSL connection with the NAE -# server. -# -# You should provide the path and file name of the CA certificate. The -# path can be absolute or relative to the application. Do not use quotes -# when specifying the path, even if it contains spaces. -# -# No default. -# -CA_File= - - -#[SSL/TLS Configuration] -# [Client Certificate] -# The client certificate to present to the NAE server. -# -# This value is required when client certificate authentication is enabled -# on the NAE server. The certificate must be in PEM format. If this value -# is set, the certificate and private key must be present even if the NAE -# server is not configured to request a client certificate. -# -# You should provide the path and file name of the client certificate. The -# path can be absolute or relative to the application. Do not use quotes -# when specifying the path, even if it contains spaces. -# -# No default. -# -Cert_File= - - -#[SSL/TLS Configuration] -# [Client Private Key] -# The private key associated with the client certificate specified in -# Cert_File. -# -# This value is required when client certificate authentication is enabled -# on the NAE server. The client private key must be in PEM-encoded PKCS#12 -# format. If this value is set, a correctly formatted key and certificate -# must be present. -# -# You should provide the path and file name of the private key. The path -# can be absolute or relative to the application. Do not use quotes when -# specifying the path, even if it contains spaces. -# -# No default. -# -Key_File= - - -#[SSL/TLS Configuration] -# [Client Private Key Passphrase] -# The passphrase to unlock the client private key specified in Key_File. -# -# This value is required when client certificate authentication is enabled -# on the NAE server. Since the value is in the clear, this properties file -# must have its permission restricted so that it can be read only by the -# applications that are to have legitimate access to it. -# -# No default. -# -Passphrase= - -#[This flag indicates that NAE USERNAME and PASSWORD are encrypted using Passpharse utility] -#default: no -Credentials_Encrypted=no - -#[This flag indicates that client certificate passphrase and keystore password is encrypted using Passpharse utility] -#default: no -Passphrase_Encrypted=no - -#[Logging Configuration] -# [Log Level] -# The level of logging that will be performed by the client. -# -# The log level determines how verbose your client logs are. You can -# disable logging by selecting NONE; however, it is recommended that you -# set the log level to LOW. A log level of HIGH can create a very large -# log file. Set the log level to HIGH to troubleshoot configuration -# problems. -# -# Valid values: -# NONE No logging -# LOW Errors only -# MEDIUM Warnings -# HIGH Info -# DEBUG Debug -# Default: LOW -# -Log_Level=NONE - - -#[Logging Configuration] -# [Log File] -# The location of the log file the client will create. -# -# You should provide the path and file name of the log file. The path can -# be absolute or relative to the application. Do not use quotes when -# specifying the path, even if it contains spaces. -# -# No default. -# -Log_File= - - -#[Logging Configuration] -# [Log Rotation] -# The log rotation method. -# -# This value specifies how frequently the log file is rotated. -# -# Valid values: -# none - log file is not rotated -# Daily - log file is rotated once a day -# Weekly - log file is rotated once a week -# Monthly - log file is rotated once a month -# Size - log file is rotated when it exceeds Log_Size_Limit -# -# Default: Daily -# -Log_Rotation=Daily - -#[Logging Configuration] -# [Log time representation] -# -# This value specifies if timestamp should be formatted in GMT or not. -# -# Valid values: -# yes - timestamps will be formatted in GMT -# no - timestamps will not be formatted in GMT -# -# Default: no -# -Log_GMT=no - - -#[Logging Configuration] -# [Log Size] -# The maximum log file size. -# -# If Log_Rotation=Size, the log will be rotated after it reaches the -# specified size. This value is only used when Log_Rotation=Size. -# -# The size may be specified in bytes, kilobytes (using 'k' or 'K'), or -# megabytes (using 'm' or 'M'). One kilobyte is 1024 bytes, and one -# megabyte is 1048576 bytes. -# -# Default: 100k -# -Log_Size_Limit=100k - -#[SYSLOG IP] -# The IP address of SYSLOG server. -# No Default -SysLog_IP= - -#[SYSLOG Port] -# The port of SYSLOG server. -# No Default -SysLog_Port= - -#[SYSLOG Protocol] -# The Protocol of SYSLOG server Either TCP or UDP or SSL. -# In case of SSL, add CA certificate and private key into keyStore -# No Default -SysLog_Protocol= - -#[SYSLOG SSL KeyStore Location] -# Location of the keystore/truststore having SYSLOG server certificates and CA. -# It is required only in case if [SYSLOG Protocol] is SSL -# By Default we will cacert from jre_home/lib/security will be referred from -SysLog_SSLKeystore= - -#[SYSLOG SSL KeyStore Password] -# Password of the keystore/truststore having SYSLOG server certificates and CA. -# It is required only in case if [SYSLOG Protocol] is SSL -# By Default cacert's password is null for java application. -SysLog_SSLKeystorePassword= - -#[Logging Configuration] -# [Additional log4j properties] -# File containing additional log4j configuration properties. -# -# Full path to the log4j configuration file, where user can specify additional -# configuration properties in "key=value" format -# For example, to set the maximum number of backup files to keep around, -# user can set "log4jIngrian.appender.ingrian.logfileAppender.MaxBackupIndex=10" -# -# Default: none -Log_Config_Advanced= - -#[Key_non_exportable_policy] -# For non exportable key in local caching mode yes means it does cipher operation remotely -# and for no means it will not go for remote operation if key is non exportable -# default = no -Key_non_exportable_policy=no - -#[Log_MaxBackupIndex] -# Maximum number of log backup files to be stored on the disk,if log rotation is enabled -# Supports integer values. If 0 or less than -1 values are supplied then maximum 7 backup files will be created -# -# default = -1, which means infinite number of backup files -Log_MaxBackupIndex=-1 diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/src/test/java/GCPCloudFunctionTester.java b/database/snowflake/CADP-SNOW-GCP-Functions/src/test/java/GCPCloudFunctionTester.java deleted file mode 100644 index dff482a3..00000000 --- a/database/snowflake/CADP-SNOW-GCP-Functions/src/test/java/GCPCloudFunctionTester.java +++ /dev/null @@ -1,173 +0,0 @@ - - - -import javax.crypto.Cipher; - -import com.google.cloud.functions.HttpFunction; -import com.google.cloud.functions.HttpRequest; -import com.google.cloud.functions.HttpResponse; -import com.ingrian.security.nae.IngrianProvider; -import com.ingrian.security.nae.FPEParameterAndFormatSpec; -import com.ingrian.security.nae.NAEKey; -import com.ingrian.security.nae.NAESession; -import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder; -import com.ingrian.security.nae.IngrianProvider.Builder; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonPrimitive; -import java.util.logging.Logger; - -/* This test app to test the logic for a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application - * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the - * data so applications or business intelligence tools do not have to change in order to use these columns. There is no need to deploy a function to run it. -* -* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested -* for all possible data sizes and combinations of encryption algorithms and IV, etc. -* Was tested with CM 2.11 & CADP 8.13 -* For more information on CADP see link below. -https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html -* For more information on Snowflake External Functions see link below. -https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp - * - *@author mwarner - * - */ -public class GCPCloudFunctionTester implements HttpFunction { -// @Override - - private static final Logger logger = Logger.getLogger(GCPCloudFunctionTester.class.getName()); - private static final Gson gson = new Gson(); - - public static void main(String[] args) throws Exception - { - - GCPCloudFunctionTester nw2 = new GCPCloudFunctionTester(); - - String request = "{ \"data\":\r\n" + - " [\r\n" + - " [ 0, \"page\" ],\r\n" + - " [ 1, \"life, the universe, and everything\" ]\r\n" + - " ]\r\n" + - "}"; - - String response = null; - nw2.service(request, response); - - - - } - - - public void service(String request, String response) - throws Exception { - - String encdata = ""; - String tweakAlgo = null; - String tweakData = null; - String snowflakereturnstring = null; - NAESession session = null; - try { - - String keyName = "testfaas"; - String userName = System.getenv("CMUSER"); - String password = System.getenv("CMPWD"); - JsonArray snowflakedata = null; - - try { - JsonElement requestParsed = gson.fromJson(request, JsonElement.class); - JsonObject requestJson = null; - - if (requestParsed != null && requestParsed.isJsonObject()) { - requestJson = requestParsed.getAsJsonObject(); - } - - if (requestJson != null && requestJson.has("data")) { - snowflakedata = requestJson.getAsJsonArray("data"); - - } - - } catch (JsonParseException e) { - logger.severe("Error parsing JSON: " + e.getMessage()); - } - - - System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", "CADP_for_JAVA.properties"); - IngrianProvider builder = new Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build(); - //IngrianProvider builder = new Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build(); - session = NAESession.getSession(userName, password.toCharArray()); - NAEKey key = NAEKey.getSecretKey(keyName, session); - int row_number = 0; - - StringBuffer snowflakereturndata = new StringBuffer(); - // Serialization - snowflakereturndata.append("{ \"data\": ["); - String algorithm = "FPE/FF1/CARD62"; - // String algorithm = "AES/CBC/PKCS5Padding"; - FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo) - .build(); - - Cipher encryptCipher = Cipher.getInstance(algorithm, "IngrianProvider"); - - for (int i = 0; i < snowflakedata.size(); i++) { - JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); - snowflakereturndata.append("["); - for (int j = 0; j < snowflakerow.size(); j++) { - - JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); - - String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); - // get a cipher - if (j == 1) { - - // FPE example - - // initialize cipher to encrypt. - encryptCipher.init(Cipher.ENCRYPT_MODE, key, param); - // encrypt data - byte[] outbuf = encryptCipher.doFinal(sensitive.getBytes()); - encdata = new String(outbuf); - - snowflakereturndata.append(encdata); - - } else { - row_number = snowflakecolumn.getAsInt(); - snowflakereturndata.append(row_number); - snowflakereturndata.append(","); - } - - } - if ( snowflakedata.size() == 1 || i == snowflakedata.size() -1) - snowflakereturndata.append("]"); - else - snowflakereturndata.append("],"); - - } - - snowflakereturndata.append("]}"); - - snowflakereturnstring = new String(snowflakereturndata); - - - } catch (Exception e) { - // return "check exception"; - } - finally{ - if(session!=null) { - session.closeSession(); - } - } - System.out.println(snowflakereturnstring); - //response.getWriter().write(snowflakereturnstring); - - } - - -@Override -public void service(HttpRequest request, HttpResponse response) throws Exception { - // TODO Auto-generated method stub - -} -} \ No newline at end of file diff --git a/database/snowflake/README.md b/database/snowflake/README.md index 51135de1..9a49983c 100644 --- a/database/snowflake/README.md +++ b/database/snowflake/README.md @@ -1,4 +1,4 @@ # Integration with Snowflake using User-Defined Functions -CADP-SNOW-AWS-Functions is for AWS Lambda -CADP-SNOW-GCP-Functions is for GCP Functions - +CADP-SNOW-AWS-Functions is for AWS Lambda (initial versions) +Thales-Snow-AWS-UDF - is the newer AWS versions that support better error handling and fewer classes with more env variables. +Thales-Snow-GCP-UDF - GCP UDFs that support better error handling and fewer classes with more env variables. \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-AWS-UDF/pom.xml b/database/snowflake/Thales-Snow-AWS-UDF/pom.xml new file mode 100644 index 00000000..b7799774 --- /dev/null +++ b/database/snowflake/Thales-Snow-AWS-UDF/pom.xml @@ -0,0 +1,147 @@ + + 4.0.0 + Thales + Thales-Snow-AWS-UDF + 0.0.3-SNAPSHOT + + + 11 + 11 + 3.12.0 + 4.4 + 1.70 + 1.10 + + 31.1-jre + 2.9.0 + 2.17.2 + 2.17.2 + .000 + + + + + + io.github.thalescpl-io.cadp + CADP_for_JAVA + 8.15.0.001 + + + + org.apache.commons + commons-lang3 + 3.12.0 + + + com.squareup.okhttp3 + okhttp + 4.10.0 + + + javax.xml.bind + jaxb-api + 2.3.1 + + + + commons-codec + commons-codec + ${org.apache.commons.codec.version} + + + com.google.guava + guava + ${guava.version} + + + com.google.code.findbugs + jsr305 + + + com.google.errorprone + error_prone_annotations + + + com.google.guava + listenablefuture + + + com.google.j2objc + j2objc-annotations + + + org.checkerframework + checker-qual + + + + + com.google.code.gson + gson + ${gson.version} + + + org.apache.commons + commons-collections4 + 4.4 + + + commons-io + commons-io + 2.4 + + + com.amazonaws + aws-lambda-java-core + 1.2.1 + + + com.amazonaws + aws-lambda-java-events + 3.11.0 + + + com.amazonaws + aws-lambda-java-log4j2 + 1.5.1 + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.2 + + false + + + + package + + shade + + + + + + + + + + + + com.github.edwgiz + maven-shade-plugin.log4j2-cachefile-transformer + + 2.13.0 + + + + + + Thales-Snow-AWS-UDF + \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/CMUserSetHelper.java b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/CMUserSetHelper.java new file mode 100644 index 00000000..d0ab18ea --- /dev/null +++ b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/CMUserSetHelper.java @@ -0,0 +1,509 @@ +package example; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; + +import java.io.BufferedReader; +import java.io.DataOutputStream; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.security.KeyStore; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; +/* + * This app provides a number of different helper methods dealing with CM Application Data Protection UserSets. There is a method to find + * a user in a userset and another method to populate the userset from a flat file. Usersets are typically used within + * an access policy but they are not restricted to that usage. + * Was tested with CM 2.14 + * Note: This source code is only to be used for testing and proof of concepts. + * @author mwarner + * + */ + +public class CMUserSetHelper { + + static String hostnamevalidate = "yourhostname"; + + String usersetid = "716f01a6-5cab-4799-925a-6dc2d8712fc1"; + String cmIP = "yourip"; + String apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?name="; + // + username_in_userset; + String addusertouserset = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users"; + + int totalrecords = 0; + String authUrl = "https://" + cmIP + "/api/v1/auth/tokens"; + + static boolean debug = true; + int chunksize = 5; + static int CHUNKSIZEMAX = 100; + + public CMUserSetHelper(String usersetid, String cmIP) { + + this.usersetid = usersetid; + this.cmIP = cmIP; + this.apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?name="; + // this.apiUrlGetUsers = "https://" + cmIP + + // "/api/v1/data-protection/user-sets/" + usersetid + "/users?limit=" + + // this.usersetlimit + "&name="; + this.addusertouserset = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users"; + this.authUrl = "https://" + cmIP + "/api/v1/auth/tokens"; + } + + public static void main(String[] args) throws Exception { + + String username = args[0]; + String password = args[1]; + String cmip = args[2]; + String usersetid = args[3]; + String filePath = args[4]; + // CMUserSetHelper("716f01a6-5cab-4799-925a-6dc2d8712fc1","20.241.70.238"); + // CMUserSetHelper("32d89a8d-efac-4c50-9b53-f51d0c03413e", + CMUserSetHelper cmusersetHelper = new CMUserSetHelper(usersetid, cmip); + int totalrecords = 0; + + // String apiUrl = apiUrlGetUsers; + String jwthtoken = geAuthToken(cmusersetHelper.authUrl, username, password); + + String newtoken = "Bearer " + removeQuotes(jwthtoken); + + // String filePath = + // "C:\\Users\\t0185905\\workspace\\CT-VL-GCP\\src\\main\\java\\com\\example\\emailAddresses.txt"; + RandomAccessFile file = new RandomAccessFile(filePath, "r"); + if (cmusersetHelper.chunksize > CHUNKSIZEMAX) + cmusersetHelper.chunksize = CHUNKSIZEMAX; + int totoalnbrofrecords = numberOfLines(file); + if (cmusersetHelper.chunksize > totoalnbrofrecords) { + cmusersetHelper.chunksize = totoalnbrofrecords / 2; + } + totalrecords = cmusersetHelper.addAUserToUserSetFromFile(cmusersetHelper.addusertouserset, newtoken, filePath); + System.out.println("Totalrecords inserted into Userset " + cmusersetHelper.usersetid + " = " + totalrecords); + + } + + /** + * Returns an boolean if user found + *

+ * + * @param user user to find + * @param newtoken jwt token to use + * @return boolean true if found in userset + * @throws CustomException + */ + public boolean findUserInUserSet(String user, String newtoken) throws CustomException { + boolean found = false; + + String apiUrl = this.apiUrlGetUsers + user; + + apiUrl = removeQuotes(apiUrl); + + try { + URL url = new URL(apiUrl); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("GET"); + connection.setRequestProperty("Authorization", newtoken); + connection.setRequestProperty("accept", "application/json"); + + connection.setDoInput(true); + BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + + StringBuilder response = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + response.append(line); + } + reader.close(); + + Gson gson = new Gson(); + if (debug) + System.out.println("response " + response); + JsonObject input = null; + JsonElement total = null; + JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject(); + if (rootNode.isJsonObject()) { + input = rootNode.getAsJsonObject(); + if (input.isJsonObject()) { + total = input.get("total"); + } + } + JsonPrimitive column = total.getAsJsonPrimitive(); + String totalstr = column.getAsJsonPrimitive().toString(); + + Integer i = Integer.valueOf(totalstr); + + if (i > 0) { + found = true; + + } + connection.disconnect(); + } catch (Exception e) { + if (e.getMessage().contains("403")) { + throw new CustomException("1002, User Not in Application Data Protection Clients ", 1002); + } else + e.printStackTrace(); + } + + return found; + + } + + /** + * Loads users from a file to the userset. Returns an int of number of users + * added. + *

+ * + * @param url url to userset api + * @param newtoken jwt token to use + * @param filePath file to load + * @return int totalnumberofrecords added to userset + */ + + public int addAUserToUserSetFromFile(String url, String newtoken, String filePath) throws IOException { + + int totalnbrofrecords = 0; + try (BufferedReader br = new BufferedReader(new FileReader(filePath))) { + String line; + int count = 0; + StringBuilder payloadBuilder = new StringBuilder(); + + payloadBuilder.append("{\"users\": ["); + + while ((line = br.readLine()) != null) { + totalnbrofrecords++; + // Append the email address to the payload + payloadBuilder.append("\"").append(line).append("\","); + count++; + + // If 'n' records have been read, print the payload + if (count == chunksize) { + makeCMCall(payloadBuilder, newtoken, url); + // Reset payload builder and count for the next batch of records + payloadBuilder = new StringBuilder("{\"users\": ["); + count = 0; + } + } + + // If there are remaining records, print the payload + if (count > 0) { + payloadBuilder.deleteCharAt(payloadBuilder.length() - 1); // Remove the trailing comma + payloadBuilder.append("}"); + makeCMCall(payloadBuilder, newtoken, url); + if (debug) + System.out.println(payloadBuilder.toString()); + } + } catch (IOException e) { + e.printStackTrace(); + } + + return totalnbrofrecords; + + } + + /** + * Loads users from a file to the userset. Returns an int of number of users + * added. + *

+ * + * @param payloadBuilder payload for CM call that contains users to add + * @param newtoken jwt token to use + * @param url url to peform inserts + * @return int response code + */ + + public static int makeCMCall(StringBuilder payloadBuilder, String newtoken, String url) throws IOException { + + payloadBuilder.deleteCharAt(payloadBuilder.length() - 1); // Remove the trailing comma + payloadBuilder.append("]}"); + if (debug) + System.out.println(payloadBuilder.toString()); + String payload = payloadBuilder.toString(); + + // Create URL object + URL apiUrl = new URL(url); + // Create connection object + HttpURLConnection connection = (HttpURLConnection) apiUrl.openConnection(); + connection.setRequestMethod("POST"); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setDoOutput(true); + connection.setRequestProperty("Authorization", newtoken); + // Send request + OutputStream outputStream = connection.getOutputStream(); + outputStream.write(payload.getBytes()); + outputStream.flush(); + outputStream.close(); + + // Get response + int responseCode = connection.getResponseCode(); + BufferedReader reader; + if (responseCode >= 200 && responseCode < 300) { + reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + } else { + reader = new BufferedReader(new InputStreamReader(connection.getErrorStream())); + } + + // Read response + String lineresponse; + StringBuilder response = new StringBuilder(); + while ((lineresponse = reader.readLine()) != null) { + response.append(lineresponse); + } + reader.close(); + + // Print response + if (debug) { + System.out.println("Response Code: " + responseCode); + System.out.println("Response Body: " + response.toString()); + } + // Disconnect connection + connection.disconnect(); + // Reset payload builder and count for the next batch of records + payloadBuilder = new StringBuilder("{\"users\": ["); + + return responseCode; + + } + + /** + * Simple sample of showing how to load a couple of users to the userset. + * Returns an int of number of users added. + *

+ * + * @param url url to userset api + * @param newtoken jwt token to use + * + * @return int response code + */ + public static int addAUserToUserSet(String url, String newtoken) throws IOException { + boolean found = false; + + String payload = "{\"users\": [\"akhip@company.com\",\"user2@company.com\"]}"; + + // Create URL object + URL apiUrl = new URL(url); + // Create connection object + HttpURLConnection connection = (HttpURLConnection) apiUrl.openConnection(); + connection.setRequestMethod("POST"); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setDoOutput(true); + connection.setRequestProperty("Authorization", newtoken); + // Send request + OutputStream outputStream = connection.getOutputStream(); + outputStream.write(payload.getBytes()); + outputStream.flush(); + outputStream.close(); + + // Get response + int responseCode = connection.getResponseCode(); + BufferedReader reader; + if (responseCode >= 200 && responseCode < 300) { + reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + } else { + reader = new BufferedReader(new InputStreamReader(connection.getErrorStream())); + } + + // Read response + String line; + StringBuilder response = new StringBuilder(); + while ((line = reader.readLine()) != null) { + response.append(line); + } + reader.close(); + + // Print response + if (debug) { + System.out.println("Response Code: " + responseCode); + System.out.println("Response Body: " + response.toString()); + } + // Disconnect connection + connection.disconnect(); + + return responseCode; + + } + + public static String removeQuotes(String input) { + + // Remove double quotes from the input string + input = input.replace("\"", ""); + + return input; + } + + private static int numberOfLines(RandomAccessFile file) throws IOException { + int numberOfLines = 0; + while (file.readLine() != null) { + numberOfLines++; + } + file.seek(0); + return numberOfLines; + } + + /** + * Get JWT from CM + *

+ * + * @param apiUrl url to CM auth + * @param username username on CM + * @param pwd password on CM + * @return String jwt + */ + + public static String geAuthToken(String apiUrl, String usernb, String pwd) throws Exception + + { + + String jStr = "{\"username\":\"" + usernb + "\",\"password\":\"" + pwd + "\"}"; + disableCertValidation(); + + String totalstr = null; + try { + URL url = new URL(apiUrl); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestProperty("Content-length", String.valueOf(jStr.length())); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setRequestMethod("GET"); + connection.setDoOutput(true); + connection.setDoInput(true); + DataOutputStream output = new DataOutputStream(connection.getOutputStream()); + output.writeBytes(jStr); + output.close(); + BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + StringBuilder response = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + response.append(line); + } + reader.close(); + + JsonObject input = null; + JsonElement total = null; + JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject(); + if (rootNode.isJsonObject()) { + input = rootNode.getAsJsonObject(); + if (input.isJsonObject()) { + total = input.get("jwt"); + } + } + JsonPrimitive column = total.getAsJsonPrimitive(); + totalstr = column.getAsJsonPrimitive().toString(); + connection.disconnect(); + } catch (Exception e) { + e.printStackTrace(); + } + + return totalstr; + + } + + /** + * Get JWT from CM + *

+ * + * @param apiUrl url to CM auth + * @param username username on CM + * @param pwd password on CM + * @return String jwt + */ + + public static String geAuthToken(String apiUrl) throws Exception + + { + + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + String jStr = "{\"username\":\"" + userName + "\",\"password\":\"" + password + "\"}"; + + disableCertValidation(); + + String totalstr = null; + try { + URL url = new URL(apiUrl); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestProperty("Content-length", String.valueOf(jStr.length())); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setRequestMethod("GET"); + connection.setDoOutput(true); + connection.setDoInput(true); + DataOutputStream output = new DataOutputStream(connection.getOutputStream()); + output.writeBytes(jStr); + output.close(); + BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + StringBuilder response = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + response.append(line); + } + reader.close(); + if (debug) + System.out.println("response " + response); + JsonObject input = null; + JsonElement total = null; + JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject(); + if (rootNode.isJsonObject()) { + input = rootNode.getAsJsonObject(); + if (input.isJsonObject()) { + total = input.get("jwt"); + } + } + JsonPrimitive column = total.getAsJsonPrimitive(); + totalstr = column.getAsJsonPrimitive().toString(); + connection.disconnect(); + } catch (Exception e) { + e.printStackTrace(); + } + + return totalstr; + + } + + public static void disableCertValidation() throws Exception { + TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() { + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + + public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) { + } + + public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) { + } + } }; + + // Install the all-trusting trust manager + try { + SSLContext sc = SSLContext.getInstance("SSL"); + sc.init(null, trustAllCerts, new java.security.SecureRandom()); + HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); + } catch (Exception e) { + e.printStackTrace(); + } + + // Create all-trusting host name verifier + HostnameVerifier allHostsValid = new HostnameVerifier() { + public boolean verify(String hostname, SSLSession session) { + return true; + } + }; + + // Install the all-trusting host verifier + HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid); + } + +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/CustomException.java b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/CustomException.java new file mode 100644 index 00000000..9bf5066c --- /dev/null +++ b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/CustomException.java @@ -0,0 +1,15 @@ +package example; + + +class CustomException extends Exception { + private int errorCode; + + public CustomException(String message, int errorCode) { + super(message); + this.errorCode = errorCode; + } + + public int getErrorCode() { + return errorCode; + } +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCADPFPEBulkUDF.java b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCADPFPEBulkUDF.java new file mode 100644 index 00000000..95794c91 --- /dev/null +++ b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCADPFPEBulkUDF.java @@ -0,0 +1,511 @@ +package example; + + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestStreamHandler; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigInteger; +import java.security.spec.AlgorithmParameterSpec; +import java.util.HashMap; +import java.util.Map; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; + +import org.apache.commons.io.IOUtils; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; +import com.ingrian.security.nae.AbstractNAECipher; +import com.ingrian.security.nae.FPEParameterAndFormatSpec; +import com.ingrian.security.nae.IngrianProvider; +import com.ingrian.security.nae.NAECipher; +import com.ingrian.security.nae.NAEKey; +import com.ingrian.security.nae.NAESession; +import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder; +import com.ingrian.security.nae.IngrianProvider.Builder; + +/* This sample AWS Lambda Function is used to implement a Snowflake Database User Defined Function(UDF). /* + * It is an example of how to use Thales Cipher Trust Application Data Protection (FCADP) + * to protect sensitive data in a column. This example uses + * Format Preserve Encryption (FPE) to maintain the original format of the data + * so applications or business intelligence tools do not have to change in order + * to use these columns. +* +* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested +* for all possible data sizes and combinations of encryption algorithms and IV, etc. +* Was tested with CM 2.14 & CADP CADP 8.15.0.001 +* For more information on CADP see link below. +https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html +* For more information on Snowflake External Functions see link below. +https://docs.snowflake.com/en/sql-reference/external-functions-creating-aws + +Notes: This example uses the CADP bulk API. +Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data +element has Unicode characters then the supported size limit would be 1750 characters + +Size of spec array should be same as the number of elements if user wants to use separate spec values +for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data +index. + * + * @author mwarner + */ + +public class ThalesAWSSnowCADPFPEBulkUDF implements RequestStreamHandler { + + private static byte[][] data; + private static AlgorithmParameterSpec[] spec; + private static Integer[] snowrownbrs; + private static final String BADDATATAG = new String("9999999999999999"); + private static int BATCHLIMIT = 10000; + + // private static final Logger logger = + // Logger.getLogger(LambdaRequestStreamHandlerNbrEncyrptBulkFPE.class.getName()); + private static final Gson gson = new Gson(); + + public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID, + String userSetLookupIP) throws Exception { + + CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP); + + String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd); + String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken); + + boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken); + + return founduserinuserset; + + } + + public String checkValid(JsonArray snowrow) { + String inputdata = null; + String notvalid = "notvalid"; + if (snowrow != null && snowrow.size() > 0) { + JsonElement element = snowrow.get(1); + if (element != null && !element.isJsonNull()) { + inputdata = element.getAsString(); + if (inputdata.isEmpty() || inputdata.length() < 2) { + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("Sensitive data is null or empty."); + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("bigquerytrow is null or empty."); + inputdata = notvalid + inputdata; + } + + return inputdata; + + } + + public String formatReturnValue(int statusCode) + + { + StringBuffer snowflakereturndatasb = new StringBuffer(); + + snowflakereturndatasb.append("{ \"statusCode\":"); + snowflakereturndatasb.append(statusCode); + snowflakereturndatasb.append(","); + snowflakereturndatasb.append(" \"body\": {"); + snowflakereturndatasb.append(" \"data\": ["); + snowflakereturndatasb.append("] }}"); + System.out.println("in exception with "); + return snowflakereturndatasb.toString(); + } + + public String formatReturnValue(int statusCode, JsonArray snowflakedata, boolean error, Cipher thalesCipher, + String datatype) throws IllegalBlockSizeException, BadPaddingException { + int row_number = 0; + + String encdata = null; + String sensitive = null; + + JsonObject bodyObject = new JsonObject(); + JsonArray dataArray = new JsonArray(); + JsonArray innerDataArray = new JsonArray(); + + for (int i = 0; i < snowflakedata.size(); i++) { + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + for (int j = 0; j < snowflakerow.size(); j++) { + if (j == 1) { + sensitive = checkValid(snowflakerow); + + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + innerDataArray.add(sensitive); + // Can not return number since a leading 0 will not work. + // innerDataArray.add(new BigInteger(sensitive)); + } else + innerDataArray.add(BADDATATAG); + + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + innerDataArray.add(""); + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + innerDataArray.add(sensitive); + } else { + innerDataArray.add(sensitive); + } + + } else { + if (!error) { + byte[] outbuf = thalesCipher.doFinal(sensitive.getBytes()); + encdata = new String(outbuf); + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + innerDataArray.add(encdata); + // innerDataArray.add(new BigInteger(encdata)); + } else { + innerDataArray.add(encdata); + } + } else { + encdata = sensitive; + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + innerDataArray.add(encdata); + // innerDataArray.add(new BigInteger(encdata)); + } else { + innerDataArray.add(encdata); + } + } + } + + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + row_number = snowflakecolumn.getAsInt(); + innerDataArray.add(row_number); + } + } + } + + bodyObject.add("data", dataArray); + JsonObject inputJsonObject = new JsonObject(); + String bodyString = bodyObject.toString(); + inputJsonObject.addProperty("statusCode", 200); + inputJsonObject.addProperty("body", bodyString); + + String formattedStringnew = inputJsonObject.toString(); + + return formattedStringnew; + } + + @Override + public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException { + // context.getLogger().log("Input: " + inputStream); + String input = IOUtils.toString(inputStream, "UTF-8"); + + Map encryptedErrorMapTotal = new HashMap(); + + String tweakAlgo = null; + String tweakData = null; + String snowflakereturnstring = null; + JsonObject body = null; + int statusCode = 200; + int numberofchunks = 0; + + String callerStr = null; + + // https://www.baeldung.com/java-aws-lambda + + JsonObject snowflakeinput = null; + + JsonArray snowflakedata = null; + NAESession session = null; + String keyName = "testfaas"; + // String keyName = System.getenv("CMKEYNAME"); + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be + // returned when the user above does not have access to the key and if doing a lookup in the userset + // and the user does not exist. If returnciphertextforuserwithnokeyaccess = no then an error will be + // returned to the query, else the results set will provide ciphertext. + // yes/no + String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess"); + boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes"); + + // usersetlookup = should a userset lookup be done on the user from Cloud DB? + // yes/no + String usersetlookup = System.getenv("usersetlookup"); + // usersetidincm = should be the usersetid in CM to query. + String usersetID = System.getenv("usersetidincm"); + // usersetlookupip = this is the IP address to query the userset. Currently it + // is the userset in CM but could be a memcache or other in memory db. + String userSetLookupIP = System.getenv("usersetlookupip"); + boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes"); + // datatype - data type in db/actual data format/return type. valid values are char, charint, nbr, charintchar + String datatype = System.getenv("datatype"); + // mode of operation valid values are : encrypt or decrypt + String mode = System.getenv("mode"); + + int batchsize = Integer.parseInt(System.getenv("BATCHSIZE")); + + int cipherType = 0; + if (mode.equals("encrypt")) + cipherType = Cipher.ENCRYPT_MODE; + else + cipherType = Cipher.DECRYPT_MODE; + + try { + + JsonElement rootNode = JsonParser.parseString(input).getAsJsonObject(); + if (rootNode.isJsonObject()) { + snowflakeinput = rootNode.getAsJsonObject(); + if (snowflakeinput.isJsonObject()) { + // For some reason when using snowflake it adds \n and \ to quotes in json. + // the JsonParser.parseString(input).getAsJsonObject(); is supposed to remove + // all of those + // characters but it does not do it for snowflake json. + JsonElement bodyele = snowflakeinput.get("body"); + String bodystr = bodyele.getAsString().replaceAll(System.lineSeparator(), ""); + bodystr = bodystr.replaceAll("\\\\", ""); + // System.out.println("bodystr after replace" + bodystr); + body = gson.fromJson(bodystr, JsonObject.class); + snowflakedata = body.getAsJsonArray("data"); + + JsonObject requestContext = snowflakeinput.getAsJsonObject("requestContext"); + + if (requestContext != null) { + JsonObject identity = requestContext.getAsJsonObject("identity"); + + if (identity != null) { + callerStr = identity.get("user").getAsString(); + System.out.println("user: " + callerStr); + } else { + System.out.println("Identity not found."); + } + } else { + System.out.println("Request context not found."); + } + + if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group + + boolean founduserinuserset = findUserInUserSet(callerStr, userName, password, usersetID, + userSetLookupIP); + // System.out.println("Found User " + founduserinuserset); + if (!founduserinuserset) + throw new CustomException("1001, User Not in User Set", 1001); + + } else { + usersetlookupbool = false; + } + + } else { + System.out.println("eerror"); + + } + } + System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", + "CADP_for_JAVA.properties"); + IngrianProvider builder = new Builder().addConfigFileInputStream( + getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build(); + + session = NAESession.getSession(userName, password.toCharArray()); + NAEKey key = NAEKey.getSecretKey(keyName, session); + String algorithm = null; + + if (datatype.equals("char")) + algorithm = "FPE/FF1/CARD62"; + else if (datatype.equals("charint")) + algorithm = "FPE/FF1/CARD10"; + else + algorithm = "FPE/FF1/CARD10"; + + int row_number = 0; + + AbstractNAECipher thalesCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider"); + + System.out.println("Batchsize = " + batchsize); + int numberOfLines = snowflakedata.size(); + int totalRowsLeft = numberOfLines; + + if (batchsize > numberOfLines) + batchsize = numberOfLines; + if (batchsize >= BATCHLIMIT) + batchsize = BATCHLIMIT; + + FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo) + .build(); + + spec = new FPEParameterAndFormatSpec[batchsize]; + data = new byte[batchsize][]; + snowrownbrs = new Integer[batchsize]; + JsonObject bodyObject = new JsonObject(); + JsonArray dataArray = new JsonArray(); + JsonArray innerDataArray = new JsonArray(); + + thalesCipher.init(cipherType, key, spec[0]); + + int i = 0; + int count = 0; + boolean newchunk = true; + int dataIndex = 0; + int specIndex = 0; + int snowRowIndex = 0; + String sensitive = null; + + while (i < numberOfLines) { + int index = 0; + + if (newchunk) { + + if (totalRowsLeft < batchsize) { + spec = new FPEParameterAndFormatSpec[totalRowsLeft]; + data = new byte[totalRowsLeft][]; + snowrownbrs = new Integer[totalRowsLeft]; + } else { + spec = new FPEParameterAndFormatSpec[batchsize]; + data = new byte[batchsize][]; + snowrownbrs = new Integer[batchsize]; + } + newchunk = false; + } + + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + for (int j = 0; j < snowflakerow.size(); j++) { + + if (j == 1) { + + // FPE example + sensitive = checkValid(snowflakerow); + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + sensitive = BADDATATAG + sensitive; + data[dataIndex++] = sensitive.getBytes(); + // Can not return number since a leading 0 will not work. + // innerDataArray.add(new BigInteger(sensitive)); + } else + data[dataIndex++] = BADDATATAG.getBytes(); + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + data[dataIndex++] = sensitive.getBytes(); + } else { + data[dataIndex++] = sensitive.getBytes(); + } + spec[specIndex++] = param; + + } else { + data[dataIndex++] = sensitive.getBytes(); + spec[specIndex++] = param; + } + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + row_number = snowflakecolumn.getAsInt(); + snowrownbrs[snowRowIndex++] = row_number; + + } + + } + + if (count == batchsize - 1) { + + // Map to store exceptions while encryption + Map encryptedErrorMap = new HashMap(); + + byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap); + + for (Map.Entry entry : encryptedErrorMap.entrySet()) { + Integer mkey = entry.getKey(); + String mvalue = entry.getValue(); + encryptedErrorMapTotal.put(mkey, mvalue); + } + + for (int enc = 0; enc < encryptedData.length; enc++) { + + innerDataArray.add(snowrownbrs[enc]); + innerDataArray.add(new String(encryptedData[enc])); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + index++; + + } + + numberofchunks++; + newchunk = true; + count = 0; + dataIndex = 0; + specIndex = 0; + snowRowIndex = 0; + } else + count++; + + totalRowsLeft--; + i++; + } + + if (count > 0) { + numberofchunks++; + int index = 0; + Map encryptedErrorMap = new HashMap(); + byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap); + for (int enc = 0; enc < encryptedData.length; enc++) { + innerDataArray.add(snowrownbrs[enc]); + innerDataArray.add(new String(encryptedData[enc])); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + index++; + + } + } + + bodyObject.add("data", dataArray); + JsonObject inputJsonObject = new JsonObject(); + String bodyString = bodyObject.toString(); + inputJsonObject.addProperty("statusCode", 200); + inputJsonObject.addProperty("body", bodyString); + + snowflakereturnstring = inputJsonObject.toString(); + // System.out.println("snowflakereturnstring = " + snowflakereturnstring); + + } catch ( + + Exception e) { + System.out.println("in exception with " + e.getMessage()); + if (returnciphertextbool) { + if (e.getMessage().contains("1401") + || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) { + + try { + snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, true, null, datatype); + } catch (IllegalBlockSizeException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } catch (BadPaddingException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } finally { + if (session != null) { + session.closeSession(); + } + } + System.out.println("number of chunks = " + numberofchunks); + outputStream.write(snowflakereturnstring.getBytes()); + } +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCADPFPEUDF.java b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCADPFPEUDF.java new file mode 100644 index 00000000..960d8f43 --- /dev/null +++ b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCADPFPEUDF.java @@ -0,0 +1,348 @@ +package example; + + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigInteger; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestStreamHandler; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; +import com.ingrian.security.nae.FPEParameterAndFormatSpec; +import com.ingrian.security.nae.IngrianProvider; +import com.ingrian.security.nae.NAEKey; +import com.ingrian.security.nae.NAESession; +import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder; +import com.ingrian.security.nae.IngrianProvider.Builder; + +/* This sample AWS Lambda Function is used to implement a Snowflake Database User Defined Function(UDF). /* + * It is an example of how to use Thales Cipher Trust Application Data Protection (CADP) + * to protect sensitive data in a column. This example uses + * Format Preserve Encryption (FPE) to maintain the original format of the data + * so applications or business intelligence tools do not have to change in order + * to use these columns. + * + * Note: This source code is only to be used for testing and proof of concepts. + * Not production ready code. Was not tested for all possible data sizes and + * combinations of encryption algorithms and IV, etc. Was tested with CM 2.14 & + * CADP 8.15.0.001 For more information on CADP see link below. +* For more information on CADP see link below. +https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html +* For more information on Snowflake External Functions see link below. +https://docs.snowflake.com/en/sql-reference/external-functions-creating-aws + * + */ + +public class ThalesAWSSnowCADPFPEUDF implements RequestStreamHandler { +// private static final Logger logger = Logger.getLogger(LambdaRequestStreamHandlerCharEncryptFPE.class.getName()); + private static final Gson gson = new Gson(); + private static final String BADDATATAG = new String ("9999999999999999"); + public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException { + // context.getLogger().log("Input: " + inputStream); + String input = IOUtils.toString(inputStream, "UTF-8"); + + String tweakAlgo = null; + String tweakData = null; + String snowflakereturnstring = null; + JsonObject body = null; + int statusCode = 200; + + // https://www.baeldung.com/java-aws-lambda + JsonObject snowflakeinput = null; + String callerStr = null; + JsonArray snowflakedata = null; + + NAESession session = null; + + String keyName = "testfaas"; + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be + // returned when the user above does not have access to the key and if doing a lookup in the userset + // and the user does not exist. If returnciphertextforuserwithnokeyaccess = no then an error will be + // returned to the query, else the results set will provide ciphertext. + // yes/no + String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess"); + boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes"); + + // usersetlookup = should a userset lookup be done on the user from Cloud DB? + // yes/no + String usersetlookup = System.getenv("usersetlookup"); + // usersetidincm = should be the usersetid in CM to query. + String usersetID = System.getenv("usersetidincm"); + // usersetlookupip = this is the IP address to query the userset. Currently it + // is the userset in CM but could be a memcache or other in memory db. + String userSetLookupIP = System.getenv("usersetlookupip"); + boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes"); + //datatype - data type in db/actual data format/return type. valid values are char, charint, nbr, charintchar + String datatype = System.getenv("datatype"); + //mode of operation valid values are : encrypt or decrypt + String mode = System.getenv("mode"); + + int cipherType = 0; + if (mode.equals("encrypt")) + cipherType = Cipher.ENCRYPT_MODE; + else + cipherType = Cipher.DECRYPT_MODE; + + + try { + + JsonElement rootNode = JsonParser.parseString(input).getAsJsonObject(); + if (rootNode.isJsonObject()) { + snowflakeinput = rootNode.getAsJsonObject(); + if (snowflakeinput.isJsonObject()) { + // For some reason when using snowflake it adds \n and \ to quotes in json. + // the JsonParser.parseString(input).getAsJsonObject(); is supposed to remove + // all of those + // characters but it does not do it for snowflake json. + JsonElement bodyele = snowflakeinput.get("body"); + String bodystr = bodyele.getAsString().replaceAll(System.lineSeparator(), ""); + // System.out.println("bodystr before replace" + bodystr ); + bodystr = bodystr.replaceAll("\\\\", ""); + // System.out.println("bodystr after replace" + bodystr ); + body = gson.fromJson(bodystr, JsonObject.class); + snowflakedata = body.getAsJsonArray("data"); + JsonObject requestContext = snowflakeinput.getAsJsonObject("requestContext"); + + if (requestContext != null) { + JsonObject identity = requestContext.getAsJsonObject("identity"); + + if (identity != null) { + callerStr = identity.get("user").getAsString(); + System.out.println("user: " + callerStr); + } else { + System.out.println("Identity not found."); + } + } else { + System.out.println("Request context not found."); + } + + if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group + + boolean founduserinuserset = findUserInUserSet(callerStr, userName, password, usersetID, + userSetLookupIP); + // System.out.println("Found User " + founduserinuserset); + if (!founduserinuserset) + throw new CustomException("1001, User Not in User Set", 1001); + + } else { + usersetlookupbool = false; + } + + } else { + System.out.println("eerror"); + + } + } + + // System.setProperty("com.ingrian.security.nae.NAE_IP.1", "10.20.1.9"); + System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", + "CADP_for_JAVA.properties"); + IngrianProvider builder = new Builder().addConfigFileInputStream( + getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build(); + session = NAESession.getSession(userName, password.toCharArray()); + NAEKey key = NAEKey.getSecretKey(keyName, session); + + String algorithm = null; + + if (datatype.equals("char")) + algorithm = "FPE/FF1/CARD62"; + else if (datatype.equals("charint")) + algorithm = "FPE/FF1/CARD10"; + else + algorithm = "FPE/FF1/CARD10"; + + FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo) + .build(); + /// + // ivSpec = param; + Cipher encryptCipher = Cipher.getInstance(algorithm, "IngrianProvider"); + + encryptCipher.init(cipherType, key, param); + snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, false, encryptCipher, datatype); + + } catch (Exception e) { + + System.out.println("in exception with " + e.getMessage()); + if (returnciphertextbool) { + if (e.getMessage().contains("1401") + || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) { + + try { + snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, true, null, datatype); + } catch (IllegalBlockSizeException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } catch (BadPaddingException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + + } finally { + if (session != null) { + session.closeSession(); + } + } + + outputStream.write(snowflakereturnstring.getBytes()); + + } + + public String formatReturnValue(int statusCode) + + { + StringBuffer snowflakereturndatasb = new StringBuffer(); + + snowflakereturndatasb.append("{ \"statusCode\":"); + snowflakereturndatasb.append(statusCode); + snowflakereturndatasb.append(","); + snowflakereturndatasb.append(" \"body\": {"); + snowflakereturndatasb.append(" \"data\": ["); + snowflakereturndatasb.append("] }}"); + System.out.println("in exception with "); + return snowflakereturndatasb.toString(); + } + + public String formatReturnValue(int statusCode, JsonArray snowflakedata, boolean error, Cipher thalesCipher, + String datatype) throws IllegalBlockSizeException, BadPaddingException { + int row_number = 0; + + String encdata = null; + String sensitive = null; + + JsonObject bodyObject = new JsonObject(); + JsonArray dataArray = new JsonArray(); + JsonArray innerDataArray = new JsonArray(); + + for (int i = 0; i < snowflakedata.size(); i++) { + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + for (int j = 0; j < snowflakerow.size(); j++) { + if (j == 1) { + sensitive = checkValid(snowflakerow); + + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + innerDataArray.add(sensitive); + // Can not return number since a leading 0 will not work. + // innerDataArray.add(new BigInteger(sensitive)); + } else + innerDataArray.add(BADDATATAG); + + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + innerDataArray.add(""); + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + innerDataArray.add(sensitive); + } else { + innerDataArray.add(sensitive); + } + + } else { + if (!error) { + byte[] outbuf = thalesCipher.doFinal(sensitive.getBytes()); + encdata = new String(outbuf); + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + innerDataArray.add(encdata); + // innerDataArray.add(new BigInteger(encdata)); + } else { + innerDataArray.add(encdata); + } + } else { + encdata = sensitive; + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + innerDataArray.add(encdata); + // innerDataArray.add(new BigInteger(encdata)); + } else { + innerDataArray.add(encdata); + } + } + } + + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + row_number = snowflakecolumn.getAsInt(); + innerDataArray.add(row_number); + } + } + } + + bodyObject.add("data", dataArray); + JsonObject inputJsonObject = new JsonObject(); + String bodyString = bodyObject.toString(); + inputJsonObject.addProperty("statusCode", 200); + inputJsonObject.addProperty("body", bodyString); + + String formattedStringnew = inputJsonObject.toString(); + + return formattedStringnew; + + } + + public String checkValid(JsonArray snowrow) { + String inputdata = null; + String notvalid = "notvalid"; + if (snowrow != null && snowrow.size() > 0) { + JsonElement element = snowrow.get(1); + if (element != null && !element.isJsonNull()) { + inputdata = element.getAsString(); + if (inputdata.isEmpty() || inputdata.length() < 2) { + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("Sensitive data is null or empty."); + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("bigquerytrow is null or empty."); + inputdata = notvalid + inputdata; + } + + return inputdata; + + } + + public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID, + String userSetLookupIP) throws Exception { + + CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP); + + String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd); + String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken); + + boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken); + + return founduserinuserset; + + } + +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCRDPFPEBulkUDF.java b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCRDPFPEBulkUDF.java new file mode 100644 index 00000000..4e4a83e4 --- /dev/null +++ b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCRDPFPEBulkUDF.java @@ -0,0 +1,674 @@ +package example; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestStreamHandler; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigInteger; +import java.security.spec.AlgorithmParameterSpec; +import java.util.HashMap; +import java.util.Map; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; + +import org.apache.commons.io.IOUtils; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; +import com.ingrian.security.nae.AbstractNAECipher; +import com.ingrian.security.nae.FPEParameterAndFormatSpec; +import com.ingrian.security.nae.IngrianProvider; +import com.ingrian.security.nae.NAECipher; +import com.ingrian.security.nae.NAEKey; +import com.ingrian.security.nae.NAESession; +import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder; +import com.ingrian.security.nae.IngrianProvider.Builder; + +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +/* This is a Thales CRDP UDF for Snowflake. It uses the Thales CRDP Bulk API. + * It is an example of how to use Thales CipherTrust REST Application Dataprotection (CRDP) + * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the + * data so applications or business intelligence tools do not have to change in order to use these columns. +* +* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested +* for all possible data sizes and combinations of encryption algorithms and IV, etc. +* Was tested with CM 2.14 and CRDP 1.0 +* For more information on CRDP see link below. +https://thalesdocs.com/ctp/con/crdp/latest/index.html +* For more information on Snowflake External Functions see link below. +https://docs.snowflake.com/en/sql-reference/external-functions-creating-aws + * + *@author mwarner + * + */ + +public class ThalesAWSSnowCRDPFPEBulkUDF implements RequestStreamHandler { + + private static int BATCHLIMIT = 10000; + private static final String BADDATATAG = new String("9999999999999999"); + private static final String REVEALRETURNTAG = new String("data"); + private static final String PROTECTRETURNTAG = new String("protected_data"); + + // private static final Logger logger = + // Logger.getLogger(LambdaRequestStreamHandlerNbrEncyrptBulkFPE.class.getName()); + private static final Gson gson = new Gson(); + + public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException { + // context.getLogger().log("Input: " + inputStream); + String input = IOUtils.toString(inputStream, "UTF-8"); + + Map snowErrorMap = new HashMap(); + String encdata = ""; + int error_count = 0; + int statusCode = 200; + + JsonObject bodyObject = new JsonObject(); + JsonArray dataArray = new JsonArray(); + JsonArray innerDataArray = new JsonArray(); + String snowflakereturnstring = null; + JsonObject body_input_request = null; + + int numberofchunks = 0; + + String callerStr = null; + + // https://www.baeldung.com/java-aws-lambda + + JsonObject snowflakeinput = null; + + JsonArray snowflakedata = null; + + String keyName = "testfaas"; + String crdpip = System.getenv("CRDPIP"); + // String keyName = System.getenv("CMKEYNAME"); + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned + // when the user above does not have access to the key and if doing a + // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no + // then an error will be returned to the query, else the results set will provide ciphertext. + String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess"); + // yes,no + boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes"); + // usersetlookup = should a userset lookup be done on the user from Cloud DB + // yes,no + String usersetlookup = System.getenv("usersetlookup"); + // usersetidincm = should be the usersetid in CM to query. + String usersetID = System.getenv("usersetidincm"); + // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be + // a memcache or other in memory db. + String userSetLookupIP = System.getenv("usersetlookupip"); + boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes"); + String keymetadatalocation = System.getenv("keymetadatalocation"); + String external_version_from_ext_source = System.getenv("keymetadata"); + String protection_profile = System.getenv("protection_profile"); + String mode = System.getenv("mode"); + String datatype = System.getenv("datatype"); + int batchsize = Integer.parseInt(System.getenv("BATCHSIZE")); + + String inputDataKey = null; + String outputDataKey = null; + String protectedData = null; + String externalkeymetadata = null; + String jsonBody = null; + + String jsonTagForProtectReveal = null; + + boolean bad_data = false; + if (mode.equals("protectbulk")) { + inputDataKey = "data_array"; + outputDataKey = "protected_data_array"; + jsonTagForProtectReveal = PROTECTRETURNTAG; + } else { + inputDataKey = "protected_data_array"; + outputDataKey = "data_array"; + jsonTagForProtectReveal = REVEALRETURNTAG; + } + + try { + + JsonElement rootNode = JsonParser.parseString(input).getAsJsonObject(); + if (rootNode.isJsonObject()) { + snowflakeinput = rootNode.getAsJsonObject(); + if (snowflakeinput.isJsonObject()) { + // For some reason when using snowflake it adds \n and \ to quotes in json. + // the JsonParser.parseString(input).getAsJsonObject(); is supposed to remove + // all of those + // characters but it does not do it for snowflake json. + JsonElement bodyele = snowflakeinput.get("body"); + String bodystr = bodyele.getAsString().replaceAll(System.lineSeparator(), ""); + bodystr = bodystr.replaceAll("\\\\", ""); + // System.out.println("bodystr after replace" + bodystr); + body_input_request = gson.fromJson(bodystr, JsonObject.class); + snowflakedata = body_input_request.getAsJsonArray("data"); + + JsonObject requestContext = snowflakeinput.getAsJsonObject("requestContext"); + + if (requestContext != null) { + JsonObject identity = requestContext.getAsJsonObject("identity"); + + if (identity != null) { + callerStr = identity.get("user").getAsString(); + System.out.println("user: " + callerStr); + } else { + System.out.println("Identity not found."); + } + } else { + System.out.println("Request context not found."); + } + + if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group + + boolean founduserinuserset = findUserInUserSet(callerStr, userName, password, usersetID, + userSetLookupIP); + // System.out.println("Found User " + founduserinuserset); + if (!founduserinuserset) + throw new CustomException("1001, User Not in User Set", 1001); + + } else { + usersetlookupbool = false; + } + + } else { + System.out.println("eerror"); + + } + } + + StringBuffer protection_policy_buff = new StringBuffer(); + String notvalid = "notvalid"; + + int numberOfLines = snowflakedata.size(); + int totalRowsLeft = numberOfLines; + + if (batchsize > numberOfLines) + batchsize = numberOfLines; + if (batchsize >= BATCHLIMIT) + batchsize = BATCHLIMIT; + // Serialization + + String crdpjsonBody = null; + // String external_version_from_ext_source = "1004001"; + // String external_version_from_ext_source = "1001001"; + + int i = 0; + int count = 0; + int totalcount = 0; + boolean newchunk = true; + int dataIndex = 0; // assumes index from snowflake will always be sequential. + JsonObject crdp_payload = new JsonObject(); + String sensitive = null; + JsonArray crdp_payload_array = new JsonArray(); + + OkHttpClient client = new OkHttpClient().newBuilder().build(); + MediaType mediaType = MediaType.parse("application/json"); + String urlStr = "http://" + crdpip + ":8090/v1/" + mode; + + while (i < numberOfLines) { + + for (int b = 0; b < batchsize && b < totalRowsLeft; b++) { + + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + sensitive = checkValid(snowflakerow); + protection_profile = protection_profile.trim(); + // Format the output + String formattedElement = String.format("\"protection_policy_name\" : \"%s\"", protection_profile); + protection_policy_buff.append(formattedElement); + protection_policy_buff.append(","); + + if (mode.equals("protectbulk")) { + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + // System.out.println("adding null not charint or nbr"); + sensitive = sensitive.replace("notvalid", ""); + sensitive = BADDATATAG + sensitive; + } else + sensitive = BADDATATAG; + + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + + } else if (sensitive.contains("notvalid")) { + // sensitive = sensitive.replace("notvalid", ""); + + } + encdata = sensitive; + + } + crdp_payload_array.add(sensitive); + } else { + JsonObject protectedDataObject = new JsonObject(); + protectedDataObject.addProperty("protected_data", sensitive); + if (keymetadatalocation.equalsIgnoreCase("external")) { + protectedDataObject.addProperty("external_version", external_version_from_ext_source); + } + crdp_payload_array.add(protectedDataObject); + + } + + if (count == batchsize - 1) { + crdp_payload.add(inputDataKey, crdp_payload_array); + String inputdataarray = null; + if (mode.equals("revealbulk")) { + crdp_payload.addProperty("username", callerStr); + inputdataarray = crdp_payload.toString(); + protection_policy_buff.append(inputdataarray); + jsonBody = protection_policy_buff.toString(); + jsonBody = jsonBody.replaceFirst("\\{", " "); + + } else { + inputdataarray = crdp_payload.toString(); + protection_policy_buff.append(inputdataarray); + inputdataarray = protection_policy_buff.toString(); + jsonBody = inputdataarray.replace("{", " "); + } + jsonBody = "{" + jsonBody; + + // System.out.println(jsonBody); + RequestBody body = RequestBody.create(mediaType, jsonBody); + + // System.out.println(urlStr); + Request crdp_request = new Request.Builder().url(urlStr).method("POST", body) + .addHeader("Content-Type", "application/json").build(); + Response crdp_response = client.newCall(crdp_request).execute(); + String crdpreturnstr = null; + if (crdp_response.isSuccessful()) { + // Parse JSON response + String responseBody = crdp_response.body().string(); + JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class); + JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey); + + String status = jsonObject.get("status").getAsString(); + int success_count = jsonObject.get("success_count").getAsInt(); + error_count = jsonObject.get("error_count").getAsInt(); + if (error_count > 0) + System.out.println("errors " + error_count); + + for (JsonElement element : protectedDataArray) { + + JsonObject protectedDataObject = element.getAsJsonObject(); + if (protectedDataObject.has(jsonTagForProtectReveal)) { + + protectedData = protectedDataObject.get(jsonTagForProtectReveal).getAsString(); + // System.out.println(protectedData); + + innerDataArray.add(dataIndex); + innerDataArray.add(new String(protectedData)); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + if (mode.equals("protectbulk")) { + if (keymetadatalocation.equalsIgnoreCase("external") + && mode.equalsIgnoreCase("protect")) { + externalkeymetadata = protectedDataObject.get("external_version") + .getAsString(); + // System.out.println("Protected Data ext key metadata need to store this: " + // + externalkeymetadata); + + } + } + } else if (protectedDataObject.has("error_message")) { + String errorMessage = protectedDataObject.get("error_message").getAsString(); + System.out.println("error_message: " + errorMessage); + snowErrorMap.put(i, errorMessage); + bad_data = true; + } else + System.out.println("unexpected json value from results: "); + dataIndex++; + + } + + crdp_payload_array = new JsonArray(); + protection_policy_buff = new StringBuffer(); + numberofchunks++; + totalcount = totalcount + count; + count = 0; + } else {// throw error.... + System.err.println("Request failed with status code: " + crdp_response.code()); + throw new CustomException("1010, Unexpected Error ", 1010); + } + + } else { + count++; + } + totalRowsLeft--; + i++; + } + } + if (count > 0) { + crdp_payload.add(inputDataKey, crdp_payload_array); + String inputdataarray = null; + if (mode.equals("revealbulk")) { + crdp_payload.addProperty("username", callerStr); + inputdataarray = crdp_payload.toString(); + protection_policy_buff.append(inputdataarray); + jsonBody = protection_policy_buff.toString(); + jsonBody = jsonBody.replaceFirst("\\{", " "); + + } else { + inputdataarray = crdp_payload.toString(); + protection_policy_buff.append(inputdataarray); + inputdataarray = protection_policy_buff.toString(); + jsonBody = inputdataarray.replace("{", " "); + } + jsonBody = "{" + jsonBody; + + // System.out.println(jsonBody); + RequestBody body = RequestBody.create(mediaType, jsonBody); + + // System.out.println(urlStr); + Request crdp_request = new Request.Builder().url(urlStr).method("POST", body) + .addHeader("Content-Type", "application/json").build(); + Response crdp_response = client.newCall(crdp_request).execute(); + String crdpreturnstr = null; + if (crdp_response.isSuccessful()) { + // Parse JSON response + String responseBody = crdp_response.body().string(); + JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class); + JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey); + + String status = jsonObject.get("status").getAsString(); + int success_count = jsonObject.get("success_count").getAsInt(); + error_count = jsonObject.get("error_count").getAsInt(); + if (error_count > 0) + System.out.println("errors " + error_count); + + if (mode.equals("protectbulk")) { + + for (JsonElement element : protectedDataArray) { + JsonObject protectedDataObject = element.getAsJsonObject(); + if (protectedDataObject.has("protected_data")) { + + protectedData = protectedDataObject.get("protected_data").getAsString(); + + innerDataArray.add(dataIndex); + innerDataArray.add(new String(protectedData)); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + if (keymetadatalocation.equalsIgnoreCase("external") + && mode.equalsIgnoreCase("protect")) { + externalkeymetadata = protectedDataObject.get("external_version").getAsString(); + // System.out.println("Protected Data ext key metadata need to store this: " + // + externalkeymetadata); + + } + } else if (protectedDataObject.has("error_message")) { + String errorMessage = protectedDataObject.get("error_message").getAsString(); + System.out.println("error_message: " + errorMessage); + snowErrorMap.put(i, errorMessage); + bad_data = true; + } else { + System.out.println("unexpected json value from results: "); + throw new CustomException("1010, Unexpected Error ", 1010); + } + dataIndex++; + } + } else { + // reveal logic + + for (JsonElement element : protectedDataArray) { + JsonObject protectedDataObject = element.getAsJsonObject(); + if (protectedDataObject.has("data")) { + protectedData = protectedDataObject.get("data").getAsString(); + // System.out.println(protectedData); + + innerDataArray.add(dataIndex); + innerDataArray.add(new String(protectedData)); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + } else if (protectedDataObject.has("error_message")) { + String errorMessage = protectedDataObject.get("error_message").getAsString(); + System.out.println("error_message: " + errorMessage); + snowErrorMap.put(i, errorMessage); + bad_data = true; + } else + System.out.println("unexpected json value from results: "); + dataIndex++; + } + } + + crdp_response.close(); + + numberofchunks++; + newchunk = true; + totalcount = totalcount + count; + count = 0; + + } else { + System.err.println("Request failed with status code: " + crdp_response.code()); + } + } + System.out.println("total chuncks " + numberofchunks); + + bodyObject.add("data", dataArray); + JsonObject inputJsonObject = new JsonObject(); + String bodyString = bodyObject.toString(); + inputJsonObject.addProperty("statusCode", 200); + inputJsonObject.addProperty("body", bodyString); + + snowflakereturnstring = inputJsonObject.toString(); + + } catch (Exception e) { + System.out.println("in exception with " + e.getMessage()); + snowflakereturnstring = "exception "; + if (returnciphertextbool) { + if (e.getMessage().contains("1401") + || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) { + + bodyObject = new JsonObject(); + dataArray = new JsonArray(); + innerDataArray = new JsonArray(); + + for (int i = 0; i < snowflakedata.size(); i++) { + + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + for (int j = 0; j < snowflakerow.size(); j++) { + if (j == 1) { + // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); + // FPE example + String sensitive = checkValid(snowflakerow); + + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + } else + sensitive = BADDATATAG; + + } else if (sensitive.equalsIgnoreCase("null") + || sensitive.equalsIgnoreCase("notvalid")) { + + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + + } + encdata = sensitive; + + } else { + // System.out.println("normal number data" + sensitive); + } + innerDataArray.add(sensitive); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + int row_number = snowflakecolumn.getAsInt(); + innerDataArray.add(row_number); + } + } + } + + bodyObject.add("data", dataArray); + JsonObject inputJsonObject = new JsonObject(); + String bodyString = bodyObject.toString(); + inputJsonObject.addProperty("statusCode", 200); + inputJsonObject.addProperty("body", bodyString); + + snowflakereturnstring = inputJsonObject.toString(); + // System.out.println(" new data " + snowflakereturnstring); + + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + + } finally { + + } + // System.out.println(snowflakereturnstring); + outputStream.write(snowflakereturnstring.getBytes()); + + } + + public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID, + String userSetLookupIP) throws Exception { + + CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP); + + String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd); + String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken); + + boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken); + + return founduserinuserset; + + } + + public String checkValid(JsonArray snowrow) { + String inputdata = null; + String notvalid = "notvalid"; + if (snowrow != null && snowrow.size() > 0) { + JsonElement element = snowrow.get(1); + if (element != null && !element.isJsonNull()) { + inputdata = element.getAsString(); + if (inputdata.isEmpty() || inputdata.length() < 2) { + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("Sensitive data is null or empty."); + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("bigquerytrow is null or empty."); + inputdata = notvalid + inputdata; + } + + return inputdata; + + } + + public String formatReturnValue(int statusCode) + + { + StringBuffer snowflakereturndatasb = new StringBuffer(); + + snowflakereturndatasb.append("{ \"statusCode\":"); + snowflakereturndatasb.append(statusCode); + snowflakereturndatasb.append(","); + snowflakereturndatasb.append(" \"body\": {"); + snowflakereturndatasb.append(" \"data\": ["); + snowflakereturndatasb.append("] }}"); + System.out.println("in exception with "); + return snowflakereturndatasb.toString(); + } + + public String formatReturnValue(int statusCode, JsonArray snowflakedata, boolean error, Cipher thalesCipher, + String datatype) throws IllegalBlockSizeException, BadPaddingException { + int row_number = 0; + + String encdata = null; + String sensitive = null; + + JsonObject bodyObject = new JsonObject(); + JsonArray dataArray = new JsonArray(); + JsonArray innerDataArray = new JsonArray(); + + for (int i = 0; i < snowflakedata.size(); i++) { + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + for (int j = 0; j < snowflakerow.size(); j++) { + if (j == 1) { + sensitive = checkValid(snowflakerow); + + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + innerDataArray.add(new BigInteger("9999999999999999")); + + // System.out.println("datatype charint or nbr adding big int 9999999999999999"); + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + innerDataArray.add(""); + // System.out.println("adding null not charint or nbr"); + } else { + innerDataArray.add(sensitive); + // System.out.println("adding sensitve" + sensitive); + } + + // System.out.println("not valid or null"); + } else { + if (!error) { + byte[] outbuf = thalesCipher.doFinal(sensitive.getBytes()); + encdata = new String(outbuf); + // System.out.println("normal vallid data "); + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + // System.out.println("char int or nbr adding as bigint"); + innerDataArray.add(new BigInteger(encdata)); + } else { + innerDataArray.add(encdata); + // System.out.println("Sensitive data: " + encdata); + } + } else { + encdata = sensitive; + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + // System.out.println("char int or nbr adding as bigint"); + innerDataArray.add(new BigInteger(encdata)); + } else { + innerDataArray.add(encdata); + } + } + } + + // innerDataArray.add(encdata); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + row_number = snowflakecolumn.getAsInt(); + innerDataArray.add(row_number); + } + } + } + + bodyObject.add("data", dataArray); + JsonObject inputJsonObject = new JsonObject(); + String bodyString = bodyObject.toString(); + inputJsonObject.addProperty("statusCode", 200); + inputJsonObject.addProperty("body", bodyString); + + String formattedStringnew = inputJsonObject.toString(); + + return formattedStringnew; + + } + +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCRDPFPEUDF.java b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCRDPFPEUDF.java new file mode 100644 index 00000000..5bd73737 --- /dev/null +++ b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCRDPFPEUDF.java @@ -0,0 +1,430 @@ +package example; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.IOUtils; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestStreamHandler; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; + +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +/* This is a Thales CRDP UDF for Snowflake. + * It is an example of how to use Thales CipherTrust REST Application Dataprotection (CRDP) + * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the + * data so applications or business intelligence tools do not have to change in order to use these columns. +* +* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested +* for all possible data sizes and combinations of encryption algorithms and IV, etc. +* Was tested with CM 2.14 and CRDP 1.0 +* For more information on CRDP see link below. +https://thalesdocs.com/ctp/con/crdp/latest/index.html +* For more information on Snowflake External Functions see link below. +https://docs.snowflake.com/en/sql-reference/external-functions-creating-aws + * + *@author mwarner + * + */ + +public class ThalesAWSSnowCRDPFPEUDF implements RequestStreamHandler { + + private static final Gson gson = new Gson(); + private static final String BADDATATAG = new String ("9999999999999999"); + private static final String REVEALRETURNTAG = new String("data"); + private static final String PROTECTRETURNTAG = new String("protected_data"); + + public String formatReturnValue(int statusCode) + + { + StringBuffer snowflakereturndatasb = new StringBuffer(); + + snowflakereturndatasb.append("{ \"statusCode\":"); + snowflakereturndatasb.append(statusCode); + snowflakereturndatasb.append(","); + snowflakereturndatasb.append(" \"body\": {"); + snowflakereturndatasb.append(" \"data\": ["); + snowflakereturndatasb.append("] }}"); + System.out.println("in exception with "); + return snowflakereturndatasb.toString(); + } + + public String checkValid(JsonArray snowrow) { + String inputdata = null; + String notvalid = "notvalid"; + if (snowrow != null && snowrow.size() > 0) { + JsonElement element = snowrow.get(1); + if (element != null && !element.isJsonNull()) { + inputdata = element.getAsString(); + if (inputdata.isEmpty() || inputdata.length() < 2) { + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("Sensitive data is null or empty."); + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("bigquerytrow is null or empty."); + inputdata = notvalid + inputdata; + } + + return inputdata; + + } + + @Override + public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException { + Map snowErrroMap = new HashMap(); + String encdata = ""; + String snowflakereturnstring = null; + String input = IOUtils.toString(inputStream, "UTF-8"); + //String input = inputStream; + JsonObject snowflakebody = null; + int statusCode = 200; + + // https://www.baeldung.com/java-aws-lambda + JsonObject snowflakeinput = null; + String callerStr = null; + JsonArray snowflakedata = null; + + String keyName = "testfaas"; + String crdpip = System.getenv("CRDPIP"); + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be + // returned when the user above does not have access to the key and if doing a lookup in the userset + // and the user does not exist. If returnciphertextforuserwithnokeyaccess = no then an error will be + // returned to the query, else the results set will provide ciphertext. + // yes/no + String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess"); + boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes"); + + // usersetlookup = should a userset lookup be done on the user from Cloud DB? + // yes/no + String usersetlookup = System.getenv("usersetlookup"); + // usersetidincm = should be the usersetid in CM to query. + String usersetID = System.getenv("usersetidincm"); + // usersetlookupip = this is the IP address to query the userset. Currently it + // is the userset in CM but could be a memcache or other in memory db. + String userSetLookupIP = System.getenv("usersetlookupip"); + boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes"); + // keymetadatalocation keymeta data can be internal or external + String keymetadatalocation = System.getenv("keymetadatalocation"); + // keymetadata represents a 7 digit value that contains policy version and key version. example 1001001. + // Normally would come from a database + String external_version_from_ext_source = System.getenv("keymetadata"); + // protection_profile = the protection profile to be used for protect or reveal. This is in CM under application + // data protection/protection profiles. + String protection_profile = System.getenv("protection_profile"); + // mode of operation. valid values are protect/reveal + String mode = System.getenv("mode"); + String datatype = System.getenv("datatype"); + String dataKey = null; + JsonObject bodyObject = new JsonObject(); + JsonArray dataArray = new JsonArray(); + JsonArray innerDataArray = new JsonArray(); + String jsonTagForProtectReveal = null; + + boolean bad_data = false; + if (mode.equals("protect")) { + dataKey = "data"; + jsonTagForProtectReveal = PROTECTRETURNTAG; + } else { + dataKey = "protected_data"; + jsonTagForProtectReveal = REVEALRETURNTAG; + } + + // This code is only to be used when input data contains user info. + + try { + + JsonElement rootNode = JsonParser.parseString(input).getAsJsonObject(); + if (rootNode.isJsonObject()) { + snowflakeinput = rootNode.getAsJsonObject(); + if (snowflakeinput.isJsonObject()) { + // For some reason when using snowflake it adds \n and \ to quotes in json. + // the JsonParser.parseString(input).getAsJsonObject(); is supposed to remove + // all of those + // characters but it does not do it for snowflake json. + JsonElement bodyele = snowflakeinput.get("body"); + String bodystr = bodyele.getAsString().replaceAll(System.lineSeparator(), ""); + // System.out.println("bodystr before replace" + bodystr ); + bodystr = bodystr.replaceAll("\\\\", ""); + // System.out.println("bodystr after replace" + bodystr ); + snowflakebody = gson.fromJson(bodystr, JsonObject.class); + snowflakedata = snowflakebody.getAsJsonArray("data"); + JsonObject requestContext = snowflakeinput.getAsJsonObject("requestContext"); + + if (requestContext != null) { + JsonObject identity = requestContext.getAsJsonObject("identity"); + + if (identity != null) { + callerStr = identity.get("user").getAsString(); + //String[] parts = callerStr.split(":"); + //callerStr = parts[0]; + System.out.println("user: " + callerStr); + } else { + System.out.println("Identity not found."); + } + } else { + System.out.println("Request context not found."); + } + + if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group + + boolean founduserinuserset = findUserInUserSet(callerStr, userName, password, usersetID, + userSetLookupIP); + // System.out.println("Found User " + founduserinuserset); + if (!founduserinuserset) + throw new CustomException("1001, User Not in User Set", 1001); + + } else { + usersetlookupbool = false; + } + + } else { + System.out.println("eerror"); + + } + } + + // Serialization + + String protectedData = null; + String externalkeymetadata = null; + String crdpjsonBody = null; + // String external_version_from_ext_source = "1004001"; + // String external_version_from_ext_source = "1001001"; + int row_number = 0; + JsonObject crdp_payload = new JsonObject(); + + crdp_payload.addProperty("protection_policy_name", protection_profile); + String sensitive = null; + + OkHttpClient client = new OkHttpClient().newBuilder().build(); + MediaType mediaType = MediaType.parse("application/json"); + String urlStr = "http://" + crdpip + ":8090/v1/" + mode; + + for (int i = 0; i < snowflakedata.size(); i++) { + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + for (int j = 0; j < snowflakerow.size(); j++) { + + if (j == 1) { + // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); + // FPE example + sensitive = checkValid(snowflakerow); + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + } else + sensitive = BADDATATAG; + + } else if (sensitive.equalsIgnoreCase("null") + || sensitive.equalsIgnoreCase("notvalid")) { + + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + + } + encdata = sensitive; + + } else { + + crdp_payload.addProperty(dataKey, sensitive); + if (mode.equals("reveal")) { + crdp_payload.addProperty("username", callerStr); + if (keymetadatalocation.equalsIgnoreCase("external")) { + crdp_payload.addProperty("external_version", external_version_from_ext_source); + } + } + crdpjsonBody = crdp_payload.toString(); + System.out.println(crdpjsonBody); + RequestBody body = RequestBody.create(mediaType, crdpjsonBody); + + // String urlStr = "\"http://" + cmip + ":8090/v1/" + mode+ "\""; + System.out.println(urlStr); + Request crdp_request = new Request.Builder() + .url(urlStr).method("POST", body).addHeader("Content-Type", "application/json") + .build(); + Response crdp_response = client.newCall(crdp_request).execute(); + + if (crdp_response.isSuccessful()) { + + // Parse JSON response + String responseBody = crdp_response.body().string(); + Gson gson = new Gson(); + JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class); + + if (mode.equals("protect")) { + if (jsonObject.has("protected_data")) { + protectedData = jsonObject.get("protected_data").getAsString(); + if (keymetadatalocation.equalsIgnoreCase("external") && mode.equalsIgnoreCase("protect")) { + externalkeymetadata = jsonObject.get("external_version").getAsString(); + // System.out.println("Protected Data ext key metadata need to store this: " + // + externalkeymetadata); + } + } else if (jsonObject.has("error_message")) { + String errorMessage = jsonObject.get("error_message").getAsString(); + System.out.println("error_message: " + errorMessage); + snowErrroMap.put(i, errorMessage); + bad_data = true; + } else + System.out.println("unexpected json value from results: "); + + } + + else if (jsonObject.has("data")) { + protectedData = jsonObject.get("data").getAsString(); + //System.out.println("Protected Data: " + protectedData); + } else if (jsonObject.has("error_message")) { + String errorMessage = jsonObject.get("error_message").getAsString(); + System.out.println("error_message: " + errorMessage); + snowErrroMap.put(i, errorMessage); + bad_data = true; + } else + System.out.println("unexpected json value from results: "); + } else { + System.err.println("Request failed with status code: " + crdp_response.code()); + } + + crdp_response.close(); + + encdata = protectedData; + + } + + innerDataArray.add(encdata); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + row_number = snowflakecolumn.getAsInt(); + innerDataArray.add(row_number); + } + + } + + } + + bodyObject.add("data", dataArray); + JsonObject inputJsonObject = new JsonObject(); + String bodyString = bodyObject.toString(); + inputJsonObject.addProperty("statusCode", 200); + inputJsonObject.addProperty("body", bodyString); + + snowflakereturnstring = inputJsonObject.toString(); + + } catch (Exception e) { + + System.out.println("in exception with " + e.getMessage()); + snowflakereturnstring = "exception "; + if (returnciphertextbool) { + if (e.getMessage().contains("1401") + || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) { + + bodyObject = new JsonObject(); + dataArray = new JsonArray(); + innerDataArray = new JsonArray(); + + for (int i = 0; i < snowflakedata.size(); i++) { + + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + for (int j = 0; j < snowflakerow.size(); j++) { + if (j == 1) { + // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); + // FPE example + String sensitive = checkValid(snowflakerow); + + sensitive = checkValid(snowflakerow); + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + //System.out.println("adding null not charint or nbr"); + sensitive = sensitive.replace("notvalid", ""); + } else + sensitive = BADDATATAG; + + } else if (sensitive.equalsIgnoreCase("null") + || sensitive.equalsIgnoreCase("notvalid")) { + + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + + } + encdata = sensitive; + + } else { + //System.out.println("normal number data" + sensitive); + } + innerDataArray.add(sensitive); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + int row_number = snowflakecolumn.getAsInt(); + innerDataArray.add(row_number); + } + } + } + + bodyObject.add("data", dataArray); + JsonObject inputJsonObject = new JsonObject(); + String bodyString = bodyObject.toString(); + inputJsonObject.addProperty("statusCode", 200); + inputJsonObject.addProperty("body", bodyString); + + snowflakereturnstring = inputJsonObject.toString(); + + + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + + } finally { + + } + //System.out.println("results" + snowflakereturnstring); + outputStream.write(snowflakereturnstring.getBytes()); + + } + + public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID, + String userSetLookupIP) throws Exception { + + CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP); + + String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd); + String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken); + + boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken); + + return founduserinuserset; + + } + +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/CMUserSetHelper.java b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/CMUserSetHelper.java new file mode 100644 index 00000000..e520e613 --- /dev/null +++ b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/CMUserSetHelper.java @@ -0,0 +1,509 @@ + + +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; + +import java.io.BufferedReader; +import java.io.DataOutputStream; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.security.KeyStore; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; +/* + * This app provides a number of different helper methods dealing with CM Application Data Protection UserSets. There is a method to find + * a user in a userset and another method to populate the userset from a flat file. Usersets are typically used within + * an access policy but they are not restricted to that usage. + * Was tested with CM 2.14 + * Note: This source code is only to be used for testing and proof of concepts. + * @author mwarner + * + */ + +public class CMUserSetHelper { + + static String hostnamevalidate = "yourhostname"; + + String usersetid = "716f01a6-5cab-4799-925a-6dc2d8712fc1"; + String cmIP = "yourip"; + String apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?name="; + // + username_in_userset; + String addusertouserset = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users"; + + int totalrecords = 0; + String authUrl = "https://" + cmIP + "/api/v1/auth/tokens"; + + static boolean debug = true; + int chunksize = 5; + static int CHUNKSIZEMAX = 100; + + public CMUserSetHelper(String usersetid, String cmIP) { + + this.usersetid = usersetid; + this.cmIP = cmIP; + this.apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?name="; + // this.apiUrlGetUsers = "https://" + cmIP + + // "/api/v1/data-protection/user-sets/" + usersetid + "/users?limit=" + + // this.usersetlimit + "&name="; + this.addusertouserset = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users"; + this.authUrl = "https://" + cmIP + "/api/v1/auth/tokens"; + } + + public static void main(String[] args) throws Exception { + + String username = args[0]; + String password = args[1]; + String cmip = args[2]; + String usersetid = args[3]; + String filePath = args[4]; + // CMUserSetHelper("716f01a6-5cab-4799-925a-6dc2d8712fc1","20.241.70.238"); + // CMUserSetHelper("32d89a8d-efac-4c50-9b53-f51d0c03413e", + CMUserSetHelper cmusersetHelper = new CMUserSetHelper(usersetid, cmip); + int totalrecords = 0; + + // String apiUrl = apiUrlGetUsers; + String jwthtoken = geAuthToken(cmusersetHelper.authUrl, username, password); + + String newtoken = "Bearer " + removeQuotes(jwthtoken); + + // String filePath = + // "C:\\Users\\t0185905\\workspace\\CT-VL-GCP\\src\\main\\java\\com\\example\\emailAddresses.txt"; + RandomAccessFile file = new RandomAccessFile(filePath, "r"); + if (cmusersetHelper.chunksize > CHUNKSIZEMAX) + cmusersetHelper.chunksize = CHUNKSIZEMAX; + int totoalnbrofrecords = numberOfLines(file); + if (cmusersetHelper.chunksize > totoalnbrofrecords) { + cmusersetHelper.chunksize = totoalnbrofrecords / 2; + } + totalrecords = cmusersetHelper.addAUserToUserSetFromFile(cmusersetHelper.addusertouserset, newtoken, filePath); + System.out.println("Totalrecords inserted into Userset " + cmusersetHelper.usersetid + " = " + totalrecords); + + } + + /** + * Returns an boolean if user found + *

+ * + * @param user user to find + * @param newtoken jwt token to use + * @return boolean true if found in userset + * @throws CustomException + */ + public boolean findUserInUserSet(String user, String newtoken) throws CustomException { + boolean found = false; + + String apiUrl = this.apiUrlGetUsers + user; + + apiUrl = removeQuotes(apiUrl); + + try { + URL url = new URL(apiUrl); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("GET"); + connection.setRequestProperty("Authorization", newtoken); + connection.setRequestProperty("accept", "application/json"); + + connection.setDoInput(true); + BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + + StringBuilder response = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + response.append(line); + } + reader.close(); + + Gson gson = new Gson(); + if (debug) + System.out.println("response " + response); + JsonObject input = null; + JsonElement total = null; + JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject(); + if (rootNode.isJsonObject()) { + input = rootNode.getAsJsonObject(); + if (input.isJsonObject()) { + total = input.get("total"); + } + } + JsonPrimitive column = total.getAsJsonPrimitive(); + String totalstr = column.getAsJsonPrimitive().toString(); + + Integer i = Integer.valueOf(totalstr); + + if (i > 0) { + found = true; + + } + connection.disconnect(); + } catch (Exception e) { + if (e.getMessage().contains("403")) { + throw new CustomException("1002, User Not in Application Data Protection Clients ", 1002); + } else + e.printStackTrace(); + } + + return found; + + } + + /** + * Loads users from a file to the userset. Returns an int of number of users + * added. + *

+ * + * @param url url to userset api + * @param newtoken jwt token to use + * @param filePath file to load + * @return int totalnumberofrecords added to userset + */ + + public int addAUserToUserSetFromFile(String url, String newtoken, String filePath) throws IOException { + + int totalnbrofrecords = 0; + try (BufferedReader br = new BufferedReader(new FileReader(filePath))) { + String line; + int count = 0; + StringBuilder payloadBuilder = new StringBuilder(); + + payloadBuilder.append("{\"users\": ["); + + while ((line = br.readLine()) != null) { + totalnbrofrecords++; + // Append the email address to the payload + payloadBuilder.append("\"").append(line).append("\","); + count++; + + // If 'n' records have been read, print the payload + if (count == chunksize) { + makeCMCall(payloadBuilder, newtoken, url); + // Reset payload builder and count for the next batch of records + payloadBuilder = new StringBuilder("{\"users\": ["); + count = 0; + } + } + + // If there are remaining records, print the payload + if (count > 0) { + payloadBuilder.deleteCharAt(payloadBuilder.length() - 1); // Remove the trailing comma + payloadBuilder.append("}"); + makeCMCall(payloadBuilder, newtoken, url); + if (debug) + System.out.println(payloadBuilder.toString()); + } + } catch (IOException e) { + e.printStackTrace(); + } + + return totalnbrofrecords; + + } + + /** + * Loads users from a file to the userset. Returns an int of number of users + * added. + *

+ * + * @param payloadBuilder payload for CM call that contains users to add + * @param newtoken jwt token to use + * @param url url to peform inserts + * @return int response code + */ + + public static int makeCMCall(StringBuilder payloadBuilder, String newtoken, String url) throws IOException { + + payloadBuilder.deleteCharAt(payloadBuilder.length() - 1); // Remove the trailing comma + payloadBuilder.append("]}"); + if (debug) + System.out.println(payloadBuilder.toString()); + String payload = payloadBuilder.toString(); + + // Create URL object + URL apiUrl = new URL(url); + // Create connection object + HttpURLConnection connection = (HttpURLConnection) apiUrl.openConnection(); + connection.setRequestMethod("POST"); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setDoOutput(true); + connection.setRequestProperty("Authorization", newtoken); + // Send request + OutputStream outputStream = connection.getOutputStream(); + outputStream.write(payload.getBytes()); + outputStream.flush(); + outputStream.close(); + + // Get response + int responseCode = connection.getResponseCode(); + BufferedReader reader; + if (responseCode >= 200 && responseCode < 300) { + reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + } else { + reader = new BufferedReader(new InputStreamReader(connection.getErrorStream())); + } + + // Read response + String lineresponse; + StringBuilder response = new StringBuilder(); + while ((lineresponse = reader.readLine()) != null) { + response.append(lineresponse); + } + reader.close(); + + // Print response + if (debug) { + System.out.println("Response Code: " + responseCode); + System.out.println("Response Body: " + response.toString()); + } + // Disconnect connection + connection.disconnect(); + // Reset payload builder and count for the next batch of records + payloadBuilder = new StringBuilder("{\"users\": ["); + + return responseCode; + + } + + /** + * Simple sample of showing how to load a couple of users to the userset. + * Returns an int of number of users added. + *

+ * + * @param url url to userset api + * @param newtoken jwt token to use + * + * @return int response code + */ + public static int addAUserToUserSet(String url, String newtoken) throws IOException { + boolean found = false; + + String payload = "{\"users\": [\"akhip@company.com\",\"user2@company.com\"]}"; + + // Create URL object + URL apiUrl = new URL(url); + // Create connection object + HttpURLConnection connection = (HttpURLConnection) apiUrl.openConnection(); + connection.setRequestMethod("POST"); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setDoOutput(true); + connection.setRequestProperty("Authorization", newtoken); + // Send request + OutputStream outputStream = connection.getOutputStream(); + outputStream.write(payload.getBytes()); + outputStream.flush(); + outputStream.close(); + + // Get response + int responseCode = connection.getResponseCode(); + BufferedReader reader; + if (responseCode >= 200 && responseCode < 300) { + reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + } else { + reader = new BufferedReader(new InputStreamReader(connection.getErrorStream())); + } + + // Read response + String line; + StringBuilder response = new StringBuilder(); + while ((line = reader.readLine()) != null) { + response.append(line); + } + reader.close(); + + // Print response + if (debug) { + System.out.println("Response Code: " + responseCode); + System.out.println("Response Body: " + response.toString()); + } + // Disconnect connection + connection.disconnect(); + + return responseCode; + + } + + public static String removeQuotes(String input) { + + // Remove double quotes from the input string + input = input.replace("\"", ""); + + return input; + } + + private static int numberOfLines(RandomAccessFile file) throws IOException { + int numberOfLines = 0; + while (file.readLine() != null) { + numberOfLines++; + } + file.seek(0); + return numberOfLines; + } + + /** + * Get JWT from CM + *

+ * + * @param apiUrl url to CM auth + * @param username username on CM + * @param pwd password on CM + * @return String jwt + */ + + public static String geAuthToken(String apiUrl, String usernb, String pwd) throws Exception + + { + + String jStr = "{\"username\":\"" + usernb + "\",\"password\":\"" + pwd + "\"}"; + disableCertValidation(); + + String totalstr = null; + try { + URL url = new URL(apiUrl); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestProperty("Content-length", String.valueOf(jStr.length())); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setRequestMethod("GET"); + connection.setDoOutput(true); + connection.setDoInput(true); + DataOutputStream output = new DataOutputStream(connection.getOutputStream()); + output.writeBytes(jStr); + output.close(); + BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + StringBuilder response = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + response.append(line); + } + reader.close(); + + JsonObject input = null; + JsonElement total = null; + JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject(); + if (rootNode.isJsonObject()) { + input = rootNode.getAsJsonObject(); + if (input.isJsonObject()) { + total = input.get("jwt"); + } + } + JsonPrimitive column = total.getAsJsonPrimitive(); + totalstr = column.getAsJsonPrimitive().toString(); + connection.disconnect(); + } catch (Exception e) { + e.printStackTrace(); + } + + return totalstr; + + } + + /** + * Get JWT from CM + *

+ * + * @param apiUrl url to CM auth + * @param username username on CM + * @param pwd password on CM + * @return String jwt + */ + + public static String geAuthToken(String apiUrl) throws Exception + + { + + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + String jStr = "{\"username\":\"" + userName + "\",\"password\":\"" + password + "\"}"; + + disableCertValidation(); + + String totalstr = null; + try { + URL url = new URL(apiUrl); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestProperty("Content-length", String.valueOf(jStr.length())); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setRequestMethod("GET"); + connection.setDoOutput(true); + connection.setDoInput(true); + DataOutputStream output = new DataOutputStream(connection.getOutputStream()); + output.writeBytes(jStr); + output.close(); + BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + StringBuilder response = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + response.append(line); + } + reader.close(); + if (debug) + System.out.println("response " + response); + JsonObject input = null; + JsonElement total = null; + JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject(); + if (rootNode.isJsonObject()) { + input = rootNode.getAsJsonObject(); + if (input.isJsonObject()) { + total = input.get("jwt"); + } + } + JsonPrimitive column = total.getAsJsonPrimitive(); + totalstr = column.getAsJsonPrimitive().toString(); + connection.disconnect(); + } catch (Exception e) { + e.printStackTrace(); + } + + return totalstr; + + } + + public static void disableCertValidation() throws Exception { + TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() { + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + + public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) { + } + + public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) { + } + } }; + + // Install the all-trusting trust manager + try { + SSLContext sc = SSLContext.getInstance("SSL"); + sc.init(null, trustAllCerts, new java.security.SecureRandom()); + HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); + } catch (Exception e) { + e.printStackTrace(); + } + + // Create all-trusting host name verifier + HostnameVerifier allHostsValid = new HostnameVerifier() { + public boolean verify(String hostname, SSLSession session) { + return true; + } + }; + + // Install the all-trusting host verifier + HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid); + } + +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/CustomException.java b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/CustomException.java new file mode 100644 index 00000000..44796974 --- /dev/null +++ b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/CustomException.java @@ -0,0 +1,15 @@ + + + +class CustomException extends Exception { + private int errorCode; + + public CustomException(String message, int errorCode) { + super(message); + this.errorCode = errorCode; + } + + public int getErrorCode() { + return errorCode; + } +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCADPFPEBulkUDFTester.java b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCADPFPEBulkUDFTester.java new file mode 100644 index 00000000..0db7ee18 --- /dev/null +++ b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCADPFPEBulkUDFTester.java @@ -0,0 +1,631 @@ + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestStreamHandler; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigInteger; +import java.security.spec.AlgorithmParameterSpec; +import java.util.HashMap; +import java.util.Map; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; + +import org.apache.commons.io.IOUtils; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; +import com.ingrian.security.nae.AbstractNAECipher; +import com.ingrian.security.nae.FPEParameterAndFormatSpec; +import com.ingrian.security.nae.IngrianProvider; +import com.ingrian.security.nae.NAECipher; +import com.ingrian.security.nae.NAEKey; +import com.ingrian.security.nae.NAESession; +import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder; +import com.ingrian.security.nae.IngrianProvider.Builder; + +/* This sample AWS Lambda Function is used to implement a Snowflake Database User Defined Function(UDF). /* + * It is an example of how to use Thales Cipher Trust Application Data Protection (CADP) + * to protect sensitive data in a column. This example uses + * Format Preserve Encryption (FPE) to maintain the original format of the data + * so applications or business intelligence tools do not have to change in order + * to use these columns. +* +* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested +* for all possible data sizes and combinations of encryption algorithms and IV, etc. +* Was tested with CM 2.14 & CADP CADP 8.15.0.001 +* For more information on CADP see link below. +https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html +* For more information on Snowflake External Functions see link below. +https://docs.snowflake.com/en/sql-reference/external-functions-creating-aws + +Notes: This example uses the CADP bulk API. +Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data +element has Unicode characters then the supported size limit would be 1750 characters + +Size of spec array should be same as the number of elements if user wants to use separate spec values +for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data +index. + * + * @author mwarner + */ + +public class ThalesAWSSnowCADPFPEBulkUDFTester implements RequestStreamHandler { + + private static byte[][] data; + private static AlgorithmParameterSpec[] spec; + private static Integer[] snowrownbrs; + private static final String BADDATATAG = new String ("9999999999999999"); + private static int BATCHLIMIT = 10000; + + // private static final Logger logger = + // Logger.getLogger(LambdaRequestStreamHandlerNbrEncyrptBulkFPE.class.getName()); + private static final Gson gson = new Gson(); + + public static void main(String[] args) throws Exception { + ThalesAWSSnowCADPFPEBulkUDFTester nw2 = new ThalesAWSSnowCADPFPEBulkUDFTester(); + String request = "{\r\n" + " \"resource\": \"/snowflake_proxy\",\r\n" + " \"path\": \"/snowflake_proxy\",\r\n" + + " \"httpMethod\": \"POST\",\r\n" + " \"headers\": {\r\n" + " \"Accept\": \"*/*\",\r\n" + + " \"Accept-Encoding\": \"gzip\",\r\n" + " \"Content-Encoding\": \"gzip\",\r\n" + + " \"Content-Type\": \"application/json\",\r\n" + + " \"Host\": \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\",\r\n" + + " \"sf-external-function-current-query-id\": \"01b02bdf-0001-7b58-0003-27d60056b5a6\",\r\n" + + " \"sf-external-function-format\": \"json\",\r\n" + + " \"sf-external-function-format-version\": \"1.0\",\r\n" + + " \"sf-external-function-name\": \"thales-aws-lambda-snow-cadp-encrypt-nbr\",\r\n" + + " \"sf-external-function-name-base64\": \"VEhBTEVTX0NBRFBfQVdTX1RPS0VOSVpFQ0hBUg==\",\r\n" + + " \"sf-external-function-query-batch-id\": \"01b02bdf-0001-7b58-0003-27d60056b5a6:1:1:0:0\",\r\n" + + " \"sf-external-function-return-type\": \"VARIANT\",\r\n" + + " \"sf-external-function-return-type-base64\": \"VkFSSUFOVA==\",\r\n" + + " \"sf-external-function-signature\": \"(B VARCHAR)\",\r\n" + + " \"sf-external-function-signature-base64\": \"KEIgVkFSQ0hBUik=\",\r\n" + + " \"User-Agent\": \"snowflake/1.0\",\r\n" + + " \"x-amz-content-sha256\": \"550cead700c849cd6f159aca9dd81083306bb5c919939c1e2c4f3f909f32332b\",\r\n" + + " \"x-amz-date\": \"20231107T142353Z\",\r\n" + + " \"x-amz-security-token\": \"IQoJb3JpZ2luX2VjEG8aCXVzLWVhc3QtMSJGMEQCICKJkvz9H9bTmSg3rRMNQYHnGeNzDyseSK3GxkGkxH6NAiBm/1kqxf5r25Z7154EYa93MaMOFIdMKS/b7MheulrJxCqHAwio//////////8BEAAaDDQ5NjQ4NDM0Mjc2NCIMCJwVrXJkl8t82Se4KtsC1mBjS4CLPcQEHJ+qRnw5HHbMTWx1DkJ6SeAjMWj/QypGy3yipoi20gY49xkbo4A0kzv7bty06Oa/VPp4id/hWdVEPVypuUQpyHKYdJAn/ZrnhvyqOMlKpGSg3157v1zPduOQOz9XvjpWC12NCaF3HAcj2/aH0DfCRacsAS+XwIWxNFOtrGbsbibrSJqFe2NqbvIbLU39cVLCD51cTMRMTXhdcRpU9JhXc8+I8+eWwsgpl823OrjwTwqDCnYs/9AY+yRCUDVjuLbKKQ+lXNWlaOMD6YfBzgNnjZZDbctx1lSMHG7HRHQrFIuZe0oKnVkbeF05CSxLBorfigKX8AF/Kyq5OhSnJPdTZKG6PbtWAub+JHaelZdEo3wDTMN04MkL+Nef+gV+g2vxejZiOc+v0HL0qXB1z2OLDqXfMm1zoACue3sYLqeK2jRjWqpEoPes7rU7jHOgkrWDgfkw7ZCpqgY6ngGFyKE++chqS6vGrZJ2e/HQL12UVqyxfT2NDRQZDA0gZmWnK5drpNJECXH9DeFcVqUnnDU77Ui5v6tXfUrq9cSOOYG68b2qCYiQ+TNkQPiMZNJmx4FYDL46RWtcxyvAMqaQz/5bhwf9W/Iw2QzYuSWj4FTC29H0eD1B5Frnq69Eg3A1A1bJ8m0LfDhT7gfK/rp7I2N0LSnYbhi2ksZU6A==\",\r\n" + + " \"X-Amzn-Trace-Id\": \"Root=1-654a4879-4aa55d5e4a5648a0268543d1\",\r\n" + + " \"X-Forwarded-For\": \"3.33.33.35\",\r\n" + " \"X-Forwarded-Port\": \"443\",\r\n" + + " \"X-Forwarded-Proto\": \"https\"\r\n" + " },\r\n" + " \"multiValueHeaders\": {\r\n" + + " \"Accept\": [\r\n" + " \"*/*\"\r\n" + " ],\r\n" + " \"Accept-Encoding\": [\r\n" + + " \"gzip\"\r\n" + " ],\r\n" + " \"Content-Encoding\": [\r\n" + " \"gzip\"\r\n" + + " ],\r\n" + " \"Content-Type\": [\r\n" + " \"application/json\"\r\n" + " ],\r\n" + + " \"Host\": [\r\n" + " \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\"\r\n" + " ],\r\n" + + " \"sf-external-function-current-query-id\": [\r\n" + " \"sdfsdfsdff-27d60056b5a6\"\r\n" + + " ],\r\n" + " \"sf-external-function-format\": [\r\n" + " \"json\"\r\n" + " ],\r\n" + + " \"sf-external-function-format-version\": [\r\n" + " \"1.0\"\r\n" + " ],\r\n" + + " \"sf-external-function-name\": [\r\n" + " \"THALES_CADP_AWS_R\"\r\n" + " ],\r\n" + + " \"sf-external-function-name-base64\": [\r\n" + " \"wewewewewewe==\"\r\n" + " ],\r\n" + + " \"sf-external-function-query-batch-id\": [\r\n" + + " \"01b02bdf-0001-7b58-0003-27d60056b5a6:1:1:0:0\"\r\n" + " ],\r\n" + + " \"sf-external-function-return-type\": [\r\n" + " \"VARIANT\"\r\n" + " ],\r\n" + + " \"sf-external-function-return-type-base64\": [\r\n" + " \"VkFSSUFOVA==\"\r\n" + " ],\r\n" + + " \"sf-external-function-signature\": [\r\n" + " \"(B VARCHAR)\"\r\n" + " ],\r\n" + + " \"sf-external-function-signature-base64\": [\r\n" + " \"KEIgVkFSQ0hBUik=\"\r\n" + + " ],\r\n" + " \"User-Agent\": [\r\n" + " \"snowflake/1.0\"\r\n" + " ],\r\n" + + " \"x-amz-content-sha256\": [\r\n" + " \"550cead70rtrtrtrtb5c919939c1e2c4f3f909f32332b\"\r\n" + + " ],\r\n" + " \"x-amz-date\": [\r\n" + " \"20231107T142353Z\"\r\n" + " ],\r\n" + + " \"x-amz-security-token\": [\r\n" + + " \"IQoJb3JpZ2luX2VjEG8aCXVzLWVhc3QtMSJGMEQCICKJkvz9H9bTmSg3rRMNQYHnGeNzDyseSK3GxkGkxH6NAiBm/1kqxf5r25Z7154EYa93MaMOFIdMKS/b7MheulrJxCqHAwio//////////8BEAAaDDQ5NjQ4NDM0Mjc2NCIMCJwVrXJkl8t82Se4KtsC1mBjS4CLPcQEHJ+qRnw5HHbMTWx1DkJ6SeAjMWj/QypGy3yipoi20gY49xkbo4A0kzv7bty06Oa/VPp4id/hWdVEPVypuUQpyHKYdJAn/ZrnhvyqOMlKpGSg3157v1zPduOQOz9XvjpWC12NCaF3HAcj2/aH0DfCRacsAS+XwIWxNFOtrGbsbibrSJqFe2NqbvIbLU39cVLCD51cTMRMTXhdcRpU9JhXc8+I8+eWwsgpl823OrjwTwqDCnYs/9AY+yRCUDVjuLbKKQ+lXNWlaOMD6YfBzgNnjZZDbctx1lSMHG7HRHQrFIuZe0oKnVkbeF05CSxLBorfigKX8AF/Kyq5OhSnJPdTZKG6PbtWAub+JHaelZdEo3wDTMN04MkL+Nef+gV+g2vxejZiOc+v0HL0qXB1z2OLDqXfMm1zoACue3sYLqeK2jRjWqpEoPes7rU7jHOgkrWDgfkw7ZCpqgY6ngGFyKE++chqS6vGrZJ2e/HQL12UVqyxfT2NDRQZDA0gZmWnK5drpNJECXH9DeFcVqUnnDU77Ui5v6tXfUrq9cSOOYG68b2qCYiQ+TNkQPiMZNJmx4FYDL46RWtcxyvAMqaQz/5bhwf9W/Iw2QzYuSWj4FTC29H0eD1B5Frnq69Eg3A1A1bJ8m0LfDhT7gfK/rp7I2N0LSnYbhi2ksZU6A==\"\r\n" + + " ],\r\n" + " \"X-Amzn-Trace-Id\": [\r\n" + + " \"Root=1-654a4879-4aa55d5e4a5648a0268543d1\"\r\n" + " ],\r\n" + + " \"X-Forwarded-For\": [\r\n" + " \"3.34.44.35\"\r\n" + " ],\r\n" + + " \"X-Forwarded-Port\": [\r\n" + " \"443\"\r\n" + " ],\r\n" + + " \"X-Forwarded-Proto\": [\r\n" + " \"https\"\r\n" + " ]\r\n" + " },\r\n" + + " \"queryStringParameters\": null,\r\n" + " \"multiValueQueryStringParameters\": null,\r\n" + + " \"pathParameters\": null,\r\n" + " \"stageVariables\": null,\r\n" + " \"requestContext\": {\r\n" + + " \"resourceId\": \"erw8lc\",\r\n" + " \"resourcePath\": \"/snowflake_proxy\",\r\n" + + " \"httpMethod\": \"POST\",\r\n" + " \"extendedRequestId\": \"OCBC9FLtiYcEThQ=\",\r\n" + + " \"requestTime\": \"07/Nov/2023:14:23:53 +0000\",\r\n" + + " \"path\": \"/test/snowflake_proxy\",\r\n" + " \"accountId\": \"455555555564\",\r\n" + + " \"protocol\": \"HTTP/1.1\",\r\n" + " \"stage\": \"test\",\r\n" + + " \"domainPrefix\": \"asdfsdfsfff\",\r\n" + " \"requestTimeEpoch\": 1699367033023,\r\n" + + " \"requestId\": \"f53460af-d302-4e3a-b837-f5a9785badcb\",\r\n" + " \"identity\": {\r\n" + + " \"cognitoIdentityPoolId\": null,\r\n" + " \"accountId\": \"34343434\",\r\n" + + " \"cognitoIdentityId\": null,\r\n" + " \"caller\": \"ARODFDFDFFDFDFX7G:snowflake\",\r\n" + + " \"sourceIp\": \"3.33.44.35\",\r\n" + " \"principalOrgId\": null,\r\n" + + " \"accessKey\": \"FAKE\",\r\n" + " \"cognitoAuthenticationType\": null,\r\n" + + " \"cognitoAuthenticationProvider\": null,\r\n" + + " \"userArn\": \"arn:aws:sts::496484342764:assumed-role/snowflakerole/snowflake\",\r\n" + + " \"userAgent\": \"snowflake/1.0\",\r\n" + + " \"user\": \"shawnscanlan@snowflakecomputing.com\"\r\n" + " },\r\n" + + " \"domainName\": \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\",\r\n" + + " \"apiId\": \"asdfsdfsfff\"\r\n" + " },\r\n" + + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"0\\\"]\\n,[1,\\\"null\\\"]\\n,[2,\\\"45\\\"]\\n,[3,\\\"32323232323232\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n" + + " \"isBase64Encoded\": false\r\n" + "}"; +//+ " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4545554545675\\\"]\\n,[1,\\\"124313145756\\\"]\\n,[2,\\\"584785473839\\\"]\\n,[3,\\\"32323232323232\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n" + // " \"user\": \"ADFDFDFFG:snowflake\"\r\n" + +// + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4\\\"]\\n,[1,\\\"12431-3145-756\\\"]\\n,[2,\\\"null\\\"]\\n,[3,\\\"\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n" + // nw2.handleRequest(request, null, null); + //backup + // + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"454-55545-45675\\\"]\\n,[1,\\\"1243-131457-56\\\"]\\n,[2,\\\"584785473839\\\"]\\n,[3,\\\"32323232323232\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n" + + nw2.handleRequest2(request, null, null); + } + + + public void handleRequest2(String inputStream, OutputStream outputStream, Context context) throws IOException { + // context.getLogger().log("Input: " + inputStream); + // String input = IOUtils.toString(inputStream, "UTF-8"); + String input = inputStream; + Map encryptedErrorMapTotal = new HashMap(); + + String tweakAlgo = null; + String tweakData = null; + String snowflakereturnstring = null; + JsonObject body = null; + int statusCode = 200; + int numberofchunks = 0; + + String callerStr = null; + + // https://www.baeldung.com/java-aws-lambda + + JsonObject snowflakeinput = null; + + JsonArray snowflakedata = null; + + NAESession session = null; + String keyName = "testfaas"; + // String keyName = System.getenv("CMKEYNAME"); + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be + // returned when the user above does not have access to the key and if doing a lookup in the userset + // and the user does not exist. If returnciphertextforuserwithnokeyaccess = no then an error will be + // returned to the query, else the results set will provide ciphertext. + // yes/no + String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess"); + boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes"); + + // usersetlookup = should a userset lookup be done on the user from Cloud DB? + // yes/no + String usersetlookup = System.getenv("usersetlookup"); + // usersetidincm = should be the usersetid in CM to query. + String usersetID = System.getenv("usersetidincm"); + // usersetlookupip = this is the IP address to query the userset. Currently it + // is the userset in CM but could be a memcache or other in memory db. + String userSetLookupIP = System.getenv("usersetlookupip"); + boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes"); + //datatype - data type in db/actual data format/return type. valid values are char, charint, nbr, charintchar + String datatype = System.getenv("datatype"); + //mode of operation valid values are : encrypt or decrypt + String mode = System.getenv("mode"); + + int cipherType = 0; + if (mode.equals("encrypt")) + cipherType = Cipher.ENCRYPT_MODE; + else + cipherType = Cipher.DECRYPT_MODE; + + + try { + + // int batchsize = Integer.parseInt(System.getenv("BATCHSIZE")); + + + JsonElement rootNode = JsonParser.parseString(input).getAsJsonObject(); + if (rootNode.isJsonObject()) { + snowflakeinput = rootNode.getAsJsonObject(); + if (snowflakeinput.isJsonObject()) { + // For some reason when using snowflake it adds \n and \ to quotes in json. + // the JsonParser.parseString(input).getAsJsonObject(); is supposed to remove + // all of those + // characters but it does not do it for snowflake json. + JsonElement bodyele = snowflakeinput.get("body"); + String bodystr = bodyele.getAsString().replaceAll(System.lineSeparator(), ""); + bodystr = bodystr.replaceAll("\\\\", ""); + // System.out.println("bodystr after replace" + bodystr); + body = gson.fromJson(bodystr, JsonObject.class); + snowflakedata = body.getAsJsonArray("data"); + + JsonObject requestContext = snowflakeinput.getAsJsonObject("requestContext"); + + if (requestContext != null) { + JsonObject identity = requestContext.getAsJsonObject("identity"); + + if (identity != null) { + callerStr = identity.get("user").getAsString(); + System.out.println("user: " + callerStr); + } else { + System.out.println("Identity not found."); + } + } else { + System.out.println("Request context not found."); + } + + if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group + + boolean founduserinuserset = findUserInUserSet(callerStr, userName, password, usersetID, + userSetLookupIP); + // System.out.println("Found User " + founduserinuserset); + if (!founduserinuserset) + throw new CustomException("1001, User Not in User Set", 1001); + + } else { + usersetlookupbool = false; + } + + } else { + System.out.println("eerror"); + + } + } + + System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", + "D:\\product\\Build\\CADP_for_JAVA.properties"); + + /* + * System.setProperty( + * "com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", + * "CADP_for_JAVA.properties"); IngrianProvider builder = new + * Builder().addConfigFileInputStream( + * getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")). + * build(); + */ + + session = NAESession.getSession(userName, password.toCharArray()); + NAEKey key = NAEKey.getSecretKey(keyName, session); + String algorithm = null; + + if (datatype.equals("char")) + algorithm = "FPE/FF1/CARD62"; + else if (datatype.equals("charint")) + algorithm = "FPE/FF1/CARD10"; + else + algorithm = "FPE/FF1/CARD10"; + + + int row_number = 0; + + AbstractNAECipher thalesCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider"); + int batchsize = 5; + System.out.println("Batchsize = " + batchsize); + int numberOfLines = snowflakedata.size(); + int totalRowsLeft = numberOfLines; + + if (batchsize > numberOfLines) + batchsize = numberOfLines; + if (batchsize >= BATCHLIMIT) + batchsize = BATCHLIMIT; + + FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo).build(); + + spec = new FPEParameterAndFormatSpec[batchsize]; + data = new byte[batchsize][]; + snowrownbrs = new Integer[batchsize]; + JsonObject bodyObject = new JsonObject(); + JsonArray dataArray = new JsonArray(); + JsonArray innerDataArray = new JsonArray(); + + thalesCipher.init(cipherType, key, spec[0]); + + int i = 0; + int count = 0; + boolean newchunk = true; + int dataIndex = 0; + int specIndex = 0; + int snowRowIndex = 0; + String sensitive = null; + while (i < numberOfLines) { + int index = 0; + + if (newchunk) { + + if (totalRowsLeft < batchsize) { + spec = new FPEParameterAndFormatSpec[totalRowsLeft]; + data = new byte[totalRowsLeft][]; + snowrownbrs = new Integer[totalRowsLeft]; + } else { + spec = new FPEParameterAndFormatSpec[batchsize]; + data = new byte[batchsize][]; + snowrownbrs = new Integer[batchsize]; + } + newchunk = false; + } + + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + for (int j = 0; j < snowflakerow.size(); j++) { + + if (j == 1) { + + // FPE example + sensitive = checkValid(snowflakerow); + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + sensitive = BADDATATAG+sensitive; + data[dataIndex++] = sensitive.getBytes(); + // Can not return number since a leading 0 will not work. + // innerDataArray.add(new BigInteger(sensitive)); + } else + data[dataIndex++] = BADDATATAG.getBytes(); + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + data[dataIndex++] = sensitive.getBytes(); + } else { + data[dataIndex++] = sensitive.getBytes(); + } + spec[specIndex++] = param; + + } else { + data[dataIndex++] = sensitive.getBytes(); + spec[specIndex++] = param; + } + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + row_number = snowflakecolumn.getAsInt(); + snowrownbrs[snowRowIndex++] = row_number; + + } + + } + + if (count == batchsize - 1) { + + // Map to store exceptions while encryption + Map encryptedErrorMap = new HashMap(); + + byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap); + + for (Map.Entry entry : encryptedErrorMap.entrySet()) { + Integer mkey = entry.getKey(); + String mvalue = entry.getValue(); + encryptedErrorMapTotal.put(mkey, mvalue); + } + + for (int enc = 0; enc < encryptedData.length; enc++) { + + innerDataArray.add(snowrownbrs[enc]); + innerDataArray.add(new String(encryptedData[enc])); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + index++; + + } + + numberofchunks++; + newchunk = true; + count = 0; + dataIndex = 0; + specIndex = 0; + snowRowIndex = 0; + } else + count++; + + totalRowsLeft--; + i++; + } + + if (count > 0) { + numberofchunks++; + int index = 0; + Map encryptedErrorMap = new HashMap(); + byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap); + for (int enc = 0; enc < encryptedData.length; enc++) { + innerDataArray.add(snowrownbrs[enc]); + innerDataArray.add(new String(encryptedData[enc])); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + index++; + + } + } + + + bodyObject.add("data", dataArray); + JsonObject inputJsonObject = new JsonObject(); + String bodyString = bodyObject.toString(); + inputJsonObject.addProperty("statusCode", 200); + inputJsonObject.addProperty("body", bodyString); + + snowflakereturnstring = inputJsonObject.toString(); + //System.out.println("snowflakereturnstring = " + snowflakereturnstring); + + + } catch ( + + Exception e) { + System.out.println("in exception with " + e.getMessage()); + if (returnciphertextbool) { + if (e.getMessage().contains("1401") || (e.getMessage().contains("1001") || (e.getMessage().contains("1002"))) ) { + + try { + snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, true, null,datatype); + } catch (IllegalBlockSizeException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } catch (BadPaddingException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } finally { + if (session != null) { + session.closeSession(); + } + } + System.out.println("number of chunks = " + numberofchunks); + System.out.println("snowflakereturnstring = " + snowflakereturnstring); + /* + * JsonObject inputObj = gson.fromJson(snowflakereturnstring, JsonObject.class); + * + * // Create the desired format JsonObject formattedObj = new JsonObject(); + * formattedObj.addProperty("statusCode", + * inputObj.get("statusCode").getAsInt()); + * + * JsonObject bodyObj = new JsonObject(); JsonArray dataArray = + * inputObj.getAsJsonObject("body").getAsJsonArray("data"); bodyObj.add("data", + * dataArray); + * + * formattedObj.addProperty("body", bodyObj.toString()); + * + * // Convert to formatted JSON string String formattedJson = + * gson.toJson(formattedObj); + * + * System.out.println("results" + formattedJson); + */ + + // outputStream.write(formattedJson.getBytes()); + + } + + public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID, + String userSetLookupIP) throws Exception { + + CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP); + + String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd); + String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken); + + boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken); + + return founduserinuserset; + + } + + public String checkValid(JsonArray snowrow) { + String inputdata = null; + String notvalid = "notvalid"; + if (snowrow != null && snowrow.size() > 0) { + JsonElement element = snowrow.get(1); + if (element != null && !element.isJsonNull()) { + inputdata = element.getAsString(); + if (inputdata.isEmpty() || inputdata.length() < 2) { + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("Sensitive data is null or empty."); + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("bigquerytrow is null or empty."); + inputdata = notvalid + inputdata; + } + + return inputdata; + + } + + + public String formatReturnValue(int statusCode) + + { + StringBuffer snowflakereturndatasb = new StringBuffer(); + + snowflakereturndatasb.append("{ \"statusCode\":"); + snowflakereturndatasb.append(statusCode); + snowflakereturndatasb.append(","); + snowflakereturndatasb.append(" \"body\": {"); + snowflakereturndatasb.append(" \"data\": ["); + snowflakereturndatasb.append("] }}"); + System.out.println("in exception with "); + return snowflakereturndatasb.toString(); + } + + public String formatReturnValue(int statusCode, JsonArray snowflakedata, boolean error, Cipher thalesCipher, + String datatype) throws IllegalBlockSizeException, BadPaddingException { + int row_number = 0; + + String encdata = null; + String sensitive = null; + + JsonObject bodyObject = new JsonObject(); + JsonArray dataArray = new JsonArray(); + JsonArray innerDataArray = new JsonArray(); + + for (int i = 0; i < snowflakedata.size(); i++) { + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + for (int j = 0; j < snowflakerow.size(); j++) { + if (j == 1) { + sensitive = checkValid(snowflakerow); + + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + innerDataArray.add(sensitive); + // Can not return number since a leading 0 will not work. + // innerDataArray.add(new BigInteger(sensitive)); + } else + innerDataArray.add(BADDATATAG); + + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + innerDataArray.add(""); + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + innerDataArray.add(sensitive); + } else { + innerDataArray.add(sensitive); + } + + } else { + if (!error) { + byte[] outbuf = thalesCipher.doFinal(sensitive.getBytes()); + encdata = new String(outbuf); + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + innerDataArray.add(encdata); + // innerDataArray.add(new BigInteger(encdata)); + } else { + innerDataArray.add(encdata); + } + } else { + encdata = sensitive; + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + innerDataArray.add(encdata); + // innerDataArray.add(new BigInteger(encdata)); + } else { + innerDataArray.add(encdata); + } + } + } + + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + row_number = snowflakecolumn.getAsInt(); + innerDataArray.add(row_number); + } + } + } + + bodyObject.add("data", dataArray); + JsonObject inputJsonObject = new JsonObject(); + String bodyString = bodyObject.toString(); + inputJsonObject.addProperty("statusCode", 200); + inputJsonObject.addProperty("body", bodyString); + + String formattedStringnew = inputJsonObject.toString(); + + return formattedStringnew; + + } + + @Override + public void handleRequest(InputStream input, OutputStream output, Context context) throws IOException { + // TODO Auto-generated method stub + + } + + +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCADPFPEUDFTester.java b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCADPFPEUDFTester.java new file mode 100644 index 00000000..631a7469 --- /dev/null +++ b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCADPFPEUDFTester.java @@ -0,0 +1,433 @@ + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigInteger; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestStreamHandler; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; +import com.ingrian.security.nae.FPEParameterAndFormatSpec; +import com.ingrian.security.nae.IngrianProvider; +import com.ingrian.security.nae.NAEKey; +import com.ingrian.security.nae.NAESession; +import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder; +import com.ingrian.security.nae.IngrianProvider.Builder; + +/* This sample AWS Lambda Function is used to implement a Snowflake Database User Defined Function(UDF). /* + * It is an example of how to use Thales Cipher Trust Application Data Protection (CADP) + * to protect sensitive data in a column. This example uses + * Format Preserve Encryption (FPE) to maintain the original format of the data + * so applications or business intelligence tools do not have to change in order + * to use these columns. + * +* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested +* for all possible data sizes and combinations of encryption algorithms and IV, etc. +* Was tested with CM 2.14 & CADP CADP 8.15.0.001 +* For more information on CADP see link below. +https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html +* For more information on Snowflake External Functions see link below. +https://docs.snowflake.com/en/sql-reference/external-functions-creating-aws + * + */ + +public class ThalesAWSSnowCADPFPEUDFTester implements RequestStreamHandler { +// private static final Logger logger = Logger.getLogger(LambdaRequestStreamHandlerCharEncryptFPE.class.getName()); + private static final Gson gson = new Gson(); + private static final String BADDATATAG = new String ("9999999999999999"); + public static void main(String[] args) throws Exception + + { + ThalesAWSSnowCADPFPEUDFTester nw2 = new ThalesAWSSnowCADPFPEUDFTester(); + String request = "{\r\n" + " \"resource\": \"/snowflake_proxy\",\r\n" + " \"path\": \"/snowflake_proxy\",\r\n" + + " \"httpMethod\": \"POST\",\r\n" + " \"headers\": {\r\n" + " \"Accept\": \"*/*\",\r\n" + + " \"Accept-Encoding\": \"gzip\",\r\n" + " \"Content-Encoding\": \"gzip\",\r\n" + + " \"Content-Type\": \"application/json\",\r\n" + + " \"Host\": \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\",\r\n" + + " \"sf-external-function-current-query-id\": \"01b02bdf-0001-7b58-0003-27d60056b5a6\",\r\n" + + " \"sf-external-function-format\": \"json\",\r\n" + + " \"sf-external-function-format-version\": \"1.0\",\r\n" + + " \"sf-external-function-name\": \"thales-aws-lambda-snow-cadp-encrypt-nbr\",\r\n" + + " \"sf-external-function-name-base64\": \"VEhBTEVTX0NBRFBfQVdTX1RPS0VOSVpFQ0hBUg==\",\r\n" + + " \"sf-external-function-query-batch-id\": \"01b02bdf-0001-7b58-0003-27d60056b5a6:1:1:0:0\",\r\n" + + " \"sf-external-function-return-type\": \"VARIANT\",\r\n" + + " \"sf-external-function-return-type-base64\": \"VkFSSUFOVA==\",\r\n" + + " \"sf-external-function-signature\": \"(B VARCHAR)\",\r\n" + + " \"sf-external-function-signature-base64\": \"KEIgVkFSQ0hBUik=\",\r\n" + + " \"User-Agent\": \"snowflake/1.0\",\r\n" + + " \"x-amz-content-sha256\": \"550cead700c849cd6f159aca9dd81083306bb5c919939c1e2c4f3f909f32332b\",\r\n" + + " \"x-amz-date\": \"20231107T142353Z\",\r\n" + + " \"x-amz-security-token\": \"IQoJb3JpZ2luX2VjEG8aCXVzLWVhc3QtMSJGMEQCICKJkvz9H9bTmSg3rRMNQYHnGeNzDyseSK3GxkGkxH6NAiBm/1kqxf5r25Z7154EYa93MaMOFIdMKS/b7MheulrJxCqHAwio//////////8BEAAaDDQ5NjQ4NDM0Mjc2NCIMCJwVrXJkl8t82Se4KtsC1mBjS4CLPcQEHJ+qRnw5HHbMTWx1DkJ6SeAjMWj/QypGy3yipoi20gY49xkbo4A0kzv7bty06Oa/VPp4id/hWdVEPVypuUQpyHKYdJAn/ZrnhvyqOMlKpGSg3157v1zPduOQOz9XvjpWC12NCaF3HAcj2/aH0DfCRacsAS+XwIWxNFOtrGbsbibrSJqFe2NqbvIbLU39cVLCD51cTMRMTXhdcRpU9JhXc8+I8+eWwsgpl823OrjwTwqDCnYs/9AY+yRCUDVjuLbKKQ+lXNWlaOMD6YfBzgNnjZZDbctx1lSMHG7HRHQrFIuZe0oKnVkbeF05CSxLBorfigKX8AF/Kyq5OhSnJPdTZKG6PbtWAub+JHaelZdEo3wDTMN04MkL+Nef+gV+g2vxejZiOc+v0HL0qXB1z2OLDqXfMm1zoACue3sYLqeK2jRjWqpEoPes7rU7jHOgkrWDgfkw7ZCpqgY6ngGFyKE++chqS6vGrZJ2e/HQL12UVqyxfT2NDRQZDA0gZmWnK5drpNJECXH9DeFcVqUnnDU77Ui5v6tXfUrq9cSOOYG68b2qCYiQ+TNkQPiMZNJmx4FYDL46RWtcxyvAMqaQz/5bhwf9W/Iw2QzYuSWj4FTC29H0eD1B5Frnq69Eg3A1A1bJ8m0LfDhT7gfK/rp7I2N0LSnYbhi2ksZU6A==\",\r\n" + + " \"X-Amzn-Trace-Id\": \"Root=1-654a4879-4aa55d5e4a5648a0268543d1\",\r\n" + + " \"X-Forwarded-For\": \"3.33.33.35\",\r\n" + " \"X-Forwarded-Port\": \"443\",\r\n" + + " \"X-Forwarded-Proto\": \"https\"\r\n" + " },\r\n" + " \"multiValueHeaders\": {\r\n" + + " \"Accept\": [\r\n" + " \"*/*\"\r\n" + " ],\r\n" + " \"Accept-Encoding\": [\r\n" + + " \"gzip\"\r\n" + " ],\r\n" + " \"Content-Encoding\": [\r\n" + " \"gzip\"\r\n" + + " ],\r\n" + " \"Content-Type\": [\r\n" + " \"application/json\"\r\n" + " ],\r\n" + + " \"Host\": [\r\n" + " \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\"\r\n" + " ],\r\n" + + " \"sf-external-function-current-query-id\": [\r\n" + " \"sdfsdfsdff-27d60056b5a6\"\r\n" + + " ],\r\n" + " \"sf-external-function-format\": [\r\n" + " \"json\"\r\n" + " ],\r\n" + + " \"sf-external-function-format-version\": [\r\n" + " \"1.0\"\r\n" + " ],\r\n" + + " \"sf-external-function-name\": [\r\n" + " \"THALES_CADP_AWS_R\"\r\n" + " ],\r\n" + + " \"sf-external-function-name-base64\": [\r\n" + " \"wewewewewewe==\"\r\n" + " ],\r\n" + + " \"sf-external-function-query-batch-id\": [\r\n" + + " \"01b02bdf-0001-7b58-0003-27d60056b5a6:1:1:0:0\"\r\n" + " ],\r\n" + + " \"sf-external-function-return-type\": [\r\n" + " \"VARIANT\"\r\n" + " ],\r\n" + + " \"sf-external-function-return-type-base64\": [\r\n" + " \"VkFSSUFOVA==\"\r\n" + " ],\r\n" + + " \"sf-external-function-signature\": [\r\n" + " \"(B VARCHAR)\"\r\n" + " ],\r\n" + + " \"sf-external-function-signature-base64\": [\r\n" + " \"KEIgVkFSQ0hBUik=\"\r\n" + + " ],\r\n" + " \"User-Agent\": [\r\n" + " \"snowflake/1.0\"\r\n" + " ],\r\n" + + " \"x-amz-content-sha256\": [\r\n" + " \"550cead70rtrtrtrtb5c919939c1e2c4f3f909f32332b\"\r\n" + + " ],\r\n" + " \"x-amz-date\": [\r\n" + " \"20231107T142353Z\"\r\n" + " ],\r\n" + + " \"x-amz-security-token\": [\r\n" + + " \"IQoJb3JpZ2luX2VjEG8aCXVzLWVhc3QtMSJGMEQCICKJkvz9H9bTmSg3rRMNQYHnGeNzDyseSK3GxkGkxH6NAiBm/1kqxf5r25Z7154EYa93MaMOFIdMKS/b7MheulrJxCqHAwio//////////8BEAAaDDQ5NjQ4NDM0Mjc2NCIMCJwVrXJkl8t82Se4KtsC1mBjS4CLPcQEHJ+qRnw5HHbMTWx1DkJ6SeAjMWj/QypGy3yipoi20gY49xkbo4A0kzv7bty06Oa/VPp4id/hWdVEPVypuUQpyHKYdJAn/ZrnhvyqOMlKpGSg3157v1zPduOQOz9XvjpWC12NCaF3HAcj2/aH0DfCRacsAS+XwIWxNFOtrGbsbibrSJqFe2NqbvIbLU39cVLCD51cTMRMTXhdcRpU9JhXc8+I8+eWwsgpl823OrjwTwqDCnYs/9AY+yRCUDVjuLbKKQ+lXNWlaOMD6YfBzgNnjZZDbctx1lSMHG7HRHQrFIuZe0oKnVkbeF05CSxLBorfigKX8AF/Kyq5OhSnJPdTZKG6PbtWAub+JHaelZdEo3wDTMN04MkL+Nef+gV+g2vxejZiOc+v0HL0qXB1z2OLDqXfMm1zoACue3sYLqeK2jRjWqpEoPes7rU7jHOgkrWDgfkw7ZCpqgY6ngGFyKE++chqS6vGrZJ2e/HQL12UVqyxfT2NDRQZDA0gZmWnK5drpNJECXH9DeFcVqUnnDU77Ui5v6tXfUrq9cSOOYG68b2qCYiQ+TNkQPiMZNJmx4FYDL46RWtcxyvAMqaQz/5bhwf9W/Iw2QzYuSWj4FTC29H0eD1B5Frnq69Eg3A1A1bJ8m0LfDhT7gfK/rp7I2N0LSnYbhi2ksZU6A==\"\r\n" + + " ],\r\n" + " \"X-Amzn-Trace-Id\": [\r\n" + + " \"Root=1-654a4879-4aa55d5e4a5648a0268543d1\"\r\n" + " ],\r\n" + + " \"X-Forwarded-For\": [\r\n" + " \"3.34.44.35\"\r\n" + " ],\r\n" + + " \"X-Forwarded-Port\": [\r\n" + " \"443\"\r\n" + " ],\r\n" + + " \"X-Forwarded-Proto\": [\r\n" + " \"https\"\r\n" + " ]\r\n" + " },\r\n" + + " \"queryStringParameters\": null,\r\n" + " \"multiValueQueryStringParameters\": null,\r\n" + + " \"pathParameters\": null,\r\n" + " \"stageVariables\": null,\r\n" + " \"requestContext\": {\r\n" + + " \"resourceId\": \"erw8lc\",\r\n" + " \"resourcePath\": \"/snowflake_proxy\",\r\n" + + " \"httpMethod\": \"POST\",\r\n" + " \"extendedRequestId\": \"OCBC9FLtiYcEThQ=\",\r\n" + + " \"requestTime\": \"07/Nov/2023:14:23:53 +0000\",\r\n" + + " \"path\": \"/test/snowflake_proxy\",\r\n" + " \"accountId\": \"455555555564\",\r\n" + + " \"protocol\": \"HTTP/1.1\",\r\n" + " \"stage\": \"test\",\r\n" + + " \"domainPrefix\": \"asdfsdfsfff\",\r\n" + " \"requestTimeEpoch\": 1699367033023,\r\n" + + " \"requestId\": \"f53460af-d302-4e3a-b837-f5a9785badcb\",\r\n" + " \"identity\": {\r\n" + + " \"cognitoIdentityPoolId\": null,\r\n" + " \"accountId\": \"34343434\",\r\n" + + " \"cognitoIdentityId\": null,\r\n" + " \"caller\": \"ARODFDFDFFDFDFX7G:snowflake\",\r\n" + + " \"sourceIp\": \"3.33.44.35\",\r\n" + " \"principalOrgId\": null,\r\n" + + " \"accessKey\": \"FAKE\",\r\n" + " \"cognitoAuthenticationType\": null,\r\n" + + " \"cognitoAuthenticationProvider\": null,\r\n" + + " \"userArn\": \"arn:aws:sts::496484342764:assumed-role/snowflakerole/snowflake\",\r\n" + + " \"userAgent\": \"snowflake/1.0\",\r\n" + + " \"user\": \"shawnscanlan@snowflakecomputing.com\"\r\n" + " },\r\n" + + " \"domainName\": \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\",\r\n" + + " \"apiId\": \"asdfsdfsfff\"\r\n" + " },\r\n" + + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"WL8ZUOD1kM\\\"]\\n]\\n}\",\r\n" + + " \"isBase64Encoded\": false\r\n" + "}"; + + //+ " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4545554545675\\\"]\\n,[1,\\\"124313145756\\\"]\\n,[2,\\\"584785473839\\\"]\\n,[3,\\\"32323232323232\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n" + //+ " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"markwarner\\\"]\\n]\\n}\",\r\n" + // decrypt 2508004056582 + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4545554545675\\\"]\\n]\\n}\",\r\n" + // + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4545554545675\\\"]\\n]\\n}\",\r\n" +// + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4545554-5456-75\\\"]\\n]\\n}\",\r\n" + // "body": + // "{\n\"data\":[\n[0,\"ndorgan5@sf_tuts.com\"],\n[1,\"cdevereu6@sf_tuts.co.au\"],\n[2,\"gglaserman7@sf_tuts.com\"],\n[3,\"icasemore8@sf_tuts.com\"],\n[4,\"chovie9@sf_tuts.com\"],\n[5,\"afeatherstona@sf_tuts.com\"],\n[6,\"hanneslieb@sf_tuts.com\"],\n[7,\"bciccoc@sf_tuts.com\"],\n[8,\"bdurnalld@sf_tuts.com\"],\n[9,\"kmacconnale@sf_tuts.com\"],\n[10,\"wsizeyf@sf_tuts.com\"],\n[11,\"dmcgowrang@sf_tuts.com\"],\n[12,\"cbedderh@sf_tuts.co.au\"],\n[13,\"davoryi@sf_tuts.com\"],\n[14,\"rtalmadgej@sf_tuts.co.uk\"],\n[15,\"lboissier@sf_tuts.com\"],\n[16,\"ihanks1@sf_tuts.com\"],\n[17,\"alaudham2@sf_tuts.com\"],\n[18,\"ecornner3@sf_tuts.com\"],\n[19,\"hgoolding4@sf_tuts.com\"],\n[20,\"adavidovitsk@sf_tuts.com\"],\n[21,\"vshermorel@sf_tuts.com\"],\n[22,\"rmattysm@sf_tuts.com\"],\n[23,\"soluwatoyinn@sf_tuts.com\"],\n[24,\"gbassfordo@sf_tuts.co.uk\"]\n]\n}", + // "isBase64Encoded": false + + // " \"user\": \"ADFDFDFFG:snowflake\"\r\n" + + // + " \"user\": \"shawnscanlan@snowflakecomputing.com\"\r\n" + " },\r\n" + nw2.handleRequest(request, null, null); + + } + + public void handleRequest(String inputStream, OutputStream outputStream, Context context) + throws IOException, IllegalBlockSizeException, BadPaddingException { + + String input = inputStream; + String tweakAlgo = null; + String tweakData = null; + String snowflakereturnstring = null; + JsonObject body = null; + int statusCode = 200; + + // https://www.baeldung.com/java-aws-lambda + JsonObject snowflakeinput = null; + String callerStr = null; + JsonArray snowflakedata = null; + + NAESession session = null; + + String keyName = "testfaas"; + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be + // returned when the user above does not have access to the key and if doing a lookup in the userset + // and the user does not exist. If returnciphertextforuserwithnokeyaccess = no then an error will be + // returned to the query, else the results set will provide ciphertext. + // yes/no + String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess"); + boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes"); + + // usersetlookup = should a userset lookup be done on the user from Cloud DB? + // yes/no + String usersetlookup = System.getenv("usersetlookup"); + // usersetidincm = should be the usersetid in CM to query. + String usersetID = System.getenv("usersetidincm"); + // usersetlookupip = this is the IP address to query the userset. Currently it + // is the userset in CM but could be a memcache or other in memory db. + String userSetLookupIP = System.getenv("usersetlookupip"); + boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes"); + //datatype - data type in db/actual data format/return type. valid values are char, charint, nbr, charintchar + String datatype = System.getenv("datatype"); + //mode of operation valid values are : encrypt or decrypt + String mode = System.getenv("mode"); + + int cipherType = 0; + if (mode.equals("encrypt")) + cipherType = Cipher.ENCRYPT_MODE; + else + cipherType = Cipher.DECRYPT_MODE; + + + try { + + JsonElement rootNode = JsonParser.parseString(input).getAsJsonObject(); + if (rootNode.isJsonObject()) { + snowflakeinput = rootNode.getAsJsonObject(); + if (snowflakeinput.isJsonObject()) { + // For some reason when using snowflake it adds \n and \ to quotes in json. + // the JsonParser.parseString(input).getAsJsonObject(); is supposed to remove + // all of those + // characters but it does not do it for snowflake json. + JsonElement bodyele = snowflakeinput.get("body"); + String bodystr = bodyele.getAsString().replaceAll(System.lineSeparator(), ""); + // System.out.println("bodystr before replace" + bodystr ); + bodystr = bodystr.replaceAll("\\\\", ""); + // System.out.println("bodystr after replace" + bodystr ); + body = gson.fromJson(bodystr, JsonObject.class); + snowflakedata = body.getAsJsonArray("data"); + JsonObject requestContext = snowflakeinput.getAsJsonObject("requestContext"); + + if (requestContext != null) { + JsonObject identity = requestContext.getAsJsonObject("identity"); + + if (identity != null) { + callerStr = identity.get("user").getAsString(); + System.out.println("user: " + callerStr); + } else { + System.out.println("Identity not found."); + } + } else { + System.out.println("Request context not found."); + } + + if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group + + boolean founduserinuserset = findUserInUserSet(callerStr, userName, password, usersetID, + userSetLookupIP); + // System.out.println("Found User " + founduserinuserset); + if (!founduserinuserset) + throw new CustomException("1001, User Not in User Set", 1001); + + } else { + usersetlookupbool = false; + } + + } else { + System.out.println("eerror"); + + } + } + + System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", + "D:\\product\\Build\\CADP_for_JAVA.properties"); + + // System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", + // "CADP_for_JAVA.properties"); + // IngrianProvider builder = new Builder().addConfigFileInputStream( + // getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build(); + session = NAESession.getSession(userName, password.toCharArray()); + NAEKey key = NAEKey.getSecretKey(keyName, session); + + String algorithm = null; + + if (datatype.equals("char")) + algorithm = "FPE/FF1/CARD62"; + else if (datatype.equals("charint")) + algorithm = "FPE/FF1/CARD10"; + else + algorithm = "FPE/FF1/CARD10"; + + FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo) + .build(); + /// + // ivSpec = param; + Cipher thalesCipher = Cipher.getInstance(algorithm, "IngrianProvider"); + + thalesCipher.init(cipherType, key, param); + snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, false, thalesCipher, datatype); + + } catch (Exception e) { + + System.out.println("in exception with " + e.getMessage()); + if (returnciphertextbool) { + if (e.getMessage().contains("1401") + || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) { + + snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, true, null, datatype); + + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + + } finally { + if (session != null) { + session.closeSession(); + } + } + System.out.println("results" + snowflakereturnstring); + // outputStream.write(snowflakereturnstring.getBytes()); + + } + + public String formatReturnValue(int statusCode) + + { + StringBuffer snowflakereturndatasb = new StringBuffer(); + + snowflakereturndatasb.append("{ \"statusCode\":"); + snowflakereturndatasb.append(statusCode); + snowflakereturndatasb.append(","); + snowflakereturndatasb.append(" \"body\": {"); + snowflakereturndatasb.append(" \"data\": ["); + snowflakereturndatasb.append("] }}"); + System.out.println("in exception with "); + return snowflakereturndatasb.toString(); + } + + public String checkValid(JsonArray snowrow) { + String inputdata = null; + String notvalid = "notvalid"; + if (snowrow != null && snowrow.size() > 0) { + JsonElement element = snowrow.get(1); + if (element != null && !element.isJsonNull()) { + inputdata = element.getAsString(); + if (inputdata.isEmpty() || inputdata.length() < 2) { + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("Sensitive data is null or empty."); + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("bigquerytrow is null or empty."); + inputdata = notvalid + inputdata; + } + + return inputdata; + + } + + public String formatReturnValue(int statusCode, JsonArray snowflakedata, boolean error, Cipher thalesCipher, + String datatype) throws IllegalBlockSizeException, BadPaddingException { + int row_number = 0; + + String encdata = null; + String sensitive = null; + + JsonObject bodyObject = new JsonObject(); + JsonArray dataArray = new JsonArray(); + JsonArray innerDataArray = new JsonArray(); + + for (int i = 0; i < snowflakedata.size(); i++) { + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + for (int j = 0; j < snowflakerow.size(); j++) { + if (j == 1) { + sensitive = checkValid(snowflakerow); + + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + innerDataArray.add(sensitive); + // Can not return number since a leading 0 will not work. + // innerDataArray.add(new BigInteger(sensitive)); + } else + innerDataArray.add(BADDATATAG); + + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + innerDataArray.add(""); + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + innerDataArray.add(sensitive); + } else { + innerDataArray.add(sensitive); + } + + } else { + if (!error) { + byte[] outbuf = thalesCipher.doFinal(sensitive.getBytes()); + encdata = new String(outbuf); + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + innerDataArray.add(encdata); + // innerDataArray.add(new BigInteger(encdata)); + } else { + innerDataArray.add(encdata); + } + } else { + encdata = sensitive; + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + innerDataArray.add(encdata); + // innerDataArray.add(new BigInteger(encdata)); + } else { + innerDataArray.add(encdata); + } + } + } + + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + row_number = snowflakecolumn.getAsInt(); + innerDataArray.add(row_number); + } + } + } + + bodyObject.add("data", dataArray); + JsonObject inputJsonObject = new JsonObject(); + String bodyString = bodyObject.toString(); + inputJsonObject.addProperty("statusCode", 200); + inputJsonObject.addProperty("body", bodyString); + + String formattedStringnew = inputJsonObject.toString(); + + return formattedStringnew; + + } + + @Override + public void handleRequest(InputStream input, OutputStream output, Context context) throws IOException { + // TODO Auto-generated method stub + + } + + public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID, + String userSetLookupIP) throws Exception { + + CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP); + + String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd); + String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken); + + boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken); + + return founduserinuserset; + + } + +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCRDPFPEBulkUDFTester.java b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCRDPFPEBulkUDFTester.java new file mode 100644 index 00000000..9d5ebd21 --- /dev/null +++ b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCRDPFPEBulkUDFTester.java @@ -0,0 +1,682 @@ + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestStreamHandler; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigInteger; +import java.security.spec.AlgorithmParameterSpec; +import java.util.HashMap; +import java.util.Map; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; + +import org.apache.commons.io.IOUtils; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; +import com.ingrian.security.nae.AbstractNAECipher; +import com.ingrian.security.nae.FPEParameterAndFormatSpec; +import com.ingrian.security.nae.IngrianProvider; +import com.ingrian.security.nae.NAECipher; +import com.ingrian.security.nae.NAEKey; +import com.ingrian.security.nae.NAESession; +import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder; +import com.ingrian.security.nae.IngrianProvider.Builder; + +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +/* This sample AWS Lambda Function is used to implement a Snowflake Database User Defined Function(UDF). /* + * It is an example of how to use Thales Cipher Trust Application Data Protection (CADP) + * to protect sensitive data in a column. This example uses + * Format Preserve Encryption (FPE) to maintain the original format of the data + * so applications or business intelligence tools do not have to change in order + * to use these columns. +* +* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested +* for all possible data sizes and combinations of encryption algorithms and IV, etc. +* Was tested with CM 2.14 & CADP CADP 8.15.0.001 +* For more information on CADP see link below. +https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html +* For more information on Snowflake External Functions see link below. +https://docs.snowflake.com/en/sql-reference/external-functions-creating-aws + +Notes: This example uses the CADP bulk API. +Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data +element has Unicode characters then the supported size limit would be 1750 characters + +Size of spec array should be same as the number of elements if user wants to use separate spec values +for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data +index. + * + * @author mwarner + */ + +public class ThalesAWSSnowCRDPFPEBulkUDFTester implements RequestStreamHandler { + + private static int BATCHLIMIT = 10000; + private static final String BADDATATAG = new String("9999999999999999"); + private static final String REVEALRETURNTAG = new String("data"); + private static final String PROTECTRETURNTAG = new String("protected_data"); + + // private static final Logger logger = + // Logger.getLogger(LambdaRequestStreamHandlerNbrEncyrptBulkFPE.class.getName()); + private static final Gson gson = new Gson(); + + public static void main(String[] args) throws Exception { + ThalesAWSSnowCRDPFPEBulkUDFTester nw2 = new ThalesAWSSnowCRDPFPEBulkUDFTester(); + String request = "{\r\n" + " \"resource\": \"/snowflake_proxy\",\r\n" + " \"path\": \"/snowflake_proxy\",\r\n" + + " \"httpMethod\": \"POST\",\r\n" + " \"headers\": {\r\n" + " \"Accept\": \"*/*\",\r\n" + + " \"Accept-Encoding\": \"gzip\",\r\n" + " \"Content-Encoding\": \"gzip\",\r\n" + + " \"Content-Type\": \"application/json\",\r\n" + + " \"Host\": \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\",\r\n" + + " \"sf-external-function-current-query-id\": \"01b02bdf-0001-7b58-0003-27d60056b5a6\",\r\n" + + " \"sf-external-function-format\": \"json\",\r\n" + + " \"sf-external-function-format-version\": \"1.0\",\r\n" + + " \"sf-external-function-name\": \"thales-aws-lambda-snow-cadp-encrypt-nbr\",\r\n" + + " \"sf-external-function-name-base64\": \"VEhBTEVTX0NBRFBfQVdTX1RPS0VOSVpFQ0hBUg==\",\r\n" + + " \"sf-external-function-query-batch-id\": \"01b02bdf-0001-7b58-0003-27d60056b5a6:1:1:0:0\",\r\n" + + " \"sf-external-function-return-type\": \"VARIANT\",\r\n" + + " \"sf-external-function-return-type-base64\": \"VkFSSUFOVA==\",\r\n" + + " \"sf-external-function-signature\": \"(B VARCHAR)\",\r\n" + + " \"sf-external-function-signature-base64\": \"KEIgVkFSQ0hBUik=\",\r\n" + + " \"User-Agent\": \"snowflake/1.0\",\r\n" + + " \"x-amz-content-sha256\": \"550cead700c849cd6f159aca9dd81083306bb5c919939c1e2c4f3f909f32332b\",\r\n" + + " \"x-amz-date\": \"20231107T142353Z\",\r\n" + + " \"x-amz-security-token\": \"IQoJb3JpZ2luX2VjEG8aCXVzLWVhc3QtMSJGMEQCICKJkvz9H9bTmSg3rRMNQYHnGeNzDyseSK3GxkGkxH6NAiBm/1kqxf5r25Z7154EYa93MaMOFIdMKS/b7MheulrJxCqHAwio//////////8BEAAaDDQ5NjQ4NDM0Mjc2NCIMCJwVrXJkl8t82Se4KtsC1mBjS4CLPcQEHJ+qRnw5HHbMTWx1DkJ6SeAjMWj/QypGy3yipoi20gY49xkbo4A0kzv7bty06Oa/VPp4id/hWdVEPVypuUQpyHKYdJAn/ZrnhvyqOMlKpGSg3157v1zPduOQOz9XvjpWC12NCaF3HAcj2/aH0DfCRacsAS+XwIWxNFOtrGbsbibrSJqFe2NqbvIbLU39cVLCD51cTMRMTXhdcRpU9JhXc8+I8+eWwsgpl823OrjwTwqDCnYs/9AY+yRCUDVjuLbKKQ+lXNWlaOMD6YfBzgNnjZZDbctx1lSMHG7HRHQrFIuZe0oKnVkbeF05CSxLBorfigKX8AF/Kyq5OhSnJPdTZKG6PbtWAub+JHaelZdEo3wDTMN04MkL+Nef+gV+g2vxejZiOc+v0HL0qXB1z2OLDqXfMm1zoACue3sYLqeK2jRjWqpEoPes7rU7jHOgkrWDgfkw7ZCpqgY6ngGFyKE++chqS6vGrZJ2e/HQL12UVqyxfT2NDRQZDA0gZmWnK5drpNJECXH9DeFcVqUnnDU77Ui5v6tXfUrq9cSOOYG68b2qCYiQ+TNkQPiMZNJmx4FYDL46RWtcxyvAMqaQz/5bhwf9W/Iw2QzYuSWj4FTC29H0eD1B5Frnq69Eg3A1A1bJ8m0LfDhT7gfK/rp7I2N0LSnYbhi2ksZU6A==\",\r\n" + + " \"X-Amzn-Trace-Id\": \"Root=1-654a4879-4aa55d5e4a5648a0268543d1\",\r\n" + + " \"X-Forwarded-For\": \"3.33.33.35\",\r\n" + " \"X-Forwarded-Port\": \"443\",\r\n" + + " \"X-Forwarded-Proto\": \"https\"\r\n" + " },\r\n" + " \"multiValueHeaders\": {\r\n" + + " \"Accept\": [\r\n" + " \"*/*\"\r\n" + " ],\r\n" + " \"Accept-Encoding\": [\r\n" + + " \"gzip\"\r\n" + " ],\r\n" + " \"Content-Encoding\": [\r\n" + " \"gzip\"\r\n" + + " ],\r\n" + " \"Content-Type\": [\r\n" + " \"application/json\"\r\n" + " ],\r\n" + + " \"Host\": [\r\n" + " \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\"\r\n" + " ],\r\n" + + " \"sf-external-function-current-query-id\": [\r\n" + " \"sdfsdfsdff-27d60056b5a6\"\r\n" + + " ],\r\n" + " \"sf-external-function-format\": [\r\n" + " \"json\"\r\n" + " ],\r\n" + + " \"sf-external-function-format-version\": [\r\n" + " \"1.0\"\r\n" + " ],\r\n" + + " \"sf-external-function-name\": [\r\n" + " \"THALES_CADP_AWS_R\"\r\n" + " ],\r\n" + + " \"sf-external-function-name-base64\": [\r\n" + " \"wewewewewewe==\"\r\n" + " ],\r\n" + + " \"sf-external-function-query-batch-id\": [\r\n" + + " \"01b02bdf-0001-7b58-0003-27d60056b5a6:1:1:0:0\"\r\n" + " ],\r\n" + + " \"sf-external-function-return-type\": [\r\n" + " \"VARIANT\"\r\n" + " ],\r\n" + + " \"sf-external-function-return-type-base64\": [\r\n" + " \"VkFSSUFOVA==\"\r\n" + " ],\r\n" + + " \"sf-external-function-signature\": [\r\n" + " \"(B VARCHAR)\"\r\n" + " ],\r\n" + + " \"sf-external-function-signature-base64\": [\r\n" + " \"KEIgVkFSQ0hBUik=\"\r\n" + + " ],\r\n" + " \"User-Agent\": [\r\n" + " \"snowflake/1.0\"\r\n" + " ],\r\n" + + " \"x-amz-content-sha256\": [\r\n" + " \"550cead70rtrtrtrtb5c919939c1e2c4f3f909f32332b\"\r\n" + + " ],\r\n" + " \"x-amz-date\": [\r\n" + " \"20231107T142353Z\"\r\n" + " ],\r\n" + + " \"x-amz-security-token\": [\r\n" + + " \"IQoJb3JpZ2luX2VjEG8aCXVzLWVhc3QtMSJGMEQCICKJkvz9H9bTmSg3rRMNQYHnGeNzDyseSK3GxkGkxH6NAiBm/1kqxf5r25Z7154EYa93MaMOFIdMKS/b7MheulrJxCqHAwio//////////8BEAAaDDQ5NjQ4NDM0Mjc2NCIMCJwVrXJkl8t82Se4KtsC1mBjS4CLPcQEHJ+qRnw5HHbMTWx1DkJ6SeAjMWj/QypGy3yipoi20gY49xkbo4A0kzv7bty06Oa/VPp4id/hWdVEPVypuUQpyHKYdJAn/ZrnhvyqOMlKpGSg3157v1zPduOQOz9XvjpWC12NCaF3HAcj2/aH0DfCRacsAS+XwIWxNFOtrGbsbibrSJqFe2NqbvIbLU39cVLCD51cTMRMTXhdcRpU9JhXc8+I8+eWwsgpl823OrjwTwqDCnYs/9AY+yRCUDVjuLbKKQ+lXNWlaOMD6YfBzgNnjZZDbctx1lSMHG7HRHQrFIuZe0oKnVkbeF05CSxLBorfigKX8AF/Kyq5OhSnJPdTZKG6PbtWAub+JHaelZdEo3wDTMN04MkL+Nef+gV+g2vxejZiOc+v0HL0qXB1z2OLDqXfMm1zoACue3sYLqeK2jRjWqpEoPes7rU7jHOgkrWDgfkw7ZCpqgY6ngGFyKE++chqS6vGrZJ2e/HQL12UVqyxfT2NDRQZDA0gZmWnK5drpNJECXH9DeFcVqUnnDU77Ui5v6tXfUrq9cSOOYG68b2qCYiQ+TNkQPiMZNJmx4FYDL46RWtcxyvAMqaQz/5bhwf9W/Iw2QzYuSWj4FTC29H0eD1B5Frnq69Eg3A1A1bJ8m0LfDhT7gfK/rp7I2N0LSnYbhi2ksZU6A==\"\r\n" + + " ],\r\n" + " \"X-Amzn-Trace-Id\": [\r\n" + + " \"Root=1-654a4879-4aa55d5e4a5648a0268543d1\"\r\n" + " ],\r\n" + + " \"X-Forwarded-For\": [\r\n" + " \"3.34.44.35\"\r\n" + " ],\r\n" + + " \"X-Forwarded-Port\": [\r\n" + " \"443\"\r\n" + " ],\r\n" + + " \"X-Forwarded-Proto\": [\r\n" + " \"https\"\r\n" + " ]\r\n" + " },\r\n" + + " \"queryStringParameters\": null,\r\n" + " \"multiValueQueryStringParameters\": null,\r\n" + + " \"pathParameters\": null,\r\n" + " \"stageVariables\": null,\r\n" + " \"requestContext\": {\r\n" + + " \"resourceId\": \"erw8lc\",\r\n" + " \"resourcePath\": \"/snowflake_proxy\",\r\n" + + " \"httpMethod\": \"POST\",\r\n" + " \"extendedRequestId\": \"OCBC9FLtiYcEThQ=\",\r\n" + + " \"requestTime\": \"07/Nov/2023:14:23:53 +0000\",\r\n" + + " \"path\": \"/test/snowflake_proxy\",\r\n" + " \"accountId\": \"455555555564\",\r\n" + + " \"protocol\": \"HTTP/1.1\",\r\n" + " \"stage\": \"test\",\r\n" + + " \"domainPrefix\": \"asdfsdfsfff\",\r\n" + " \"requestTimeEpoch\": 1699367033023,\r\n" + + " \"requestId\": \"f53460af-d302-4e3a-b837-f5a9785badcb\",\r\n" + " \"identity\": {\r\n" + + " \"cognitoIdentityPoolId\": null,\r\n" + " \"accountId\": \"34343434\",\r\n" + + " \"cognitoIdentityId\": null,\r\n" + " \"caller\": \"ARODFDFDFFDFDFX7G:snowflake\",\r\n" + + " \"sourceIp\": \"3.33.44.35\",\r\n" + " \"principalOrgId\": null,\r\n" + + " \"accessKey\": \"FAKE\",\r\n" + " \"cognitoAuthenticationType\": null,\r\n" + + " \"cognitoAuthenticationProvider\": null,\r\n" + + " \"userArn\": \"arn:aws:sts::496484342764:assumed-role/snowflakerole/snowflake\",\r\n" + + " \"userAgent\": \"snowflake/1.0\",\r\n" + + " \"user\": \"shawnscanlan@snowflakecomputing.com\"\r\n" + " },\r\n" + + " \"domainName\": \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\",\r\n" + + " \"apiId\": \"asdfsdfsfff\"\r\n" + " },\r\n" + + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"47764370337599674\\\"]\\n,[1,\\\"9500405729277875\\\"]\\n,[2,\\\"57\\\"]\\n,[3,\\\"29330926862189\\\"]\\n,[4,\\\"909554854973\\\"]\\n]\\n}\",\r\n" + + " \"isBase64Encoded\": false\r\n" + "}"; + +//+ " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4545554545675\\\"]\\n,[1,\\\"124313145756\\\"]\\n,[2,\\\"584785473839\\\"]\\n,[3,\\\"32323232323232\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n" +//+ " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"47764370337599674\\\"]\\n,[1,\\\"9500405729277875\\\"]\\n,[2,\\\"57\\\"]\\n,[3,\\\"64310756511368\\\"]\\n,[4,\\\"909554854973\\\"]\\n]\\n}\",\r\n" + // + " \"body\": + // \"{\\n\\\"data\\\":[\\n[0,\\\"6\\\"]\\n,[1,\\\"null\\\"]\\n,[2,\\\"49\\\"]\\n,[3,\\\"64310756511368\\\"]\\n,[4,\\\"294461560470\\\"]\\n]\\n}\",\r\n" + // temp{"data":[[0,"1676960812748"],[1,"933211143272"],[2,"505141998387"],[3,"64310756511368"],[4,"294461560470"]]} +// reveal 1002001 temp{"data":[[0,"47764370337599674"],[1,"9500405729277875"],[2,"57"],[3,"29330926862189"],[4,"909554854973"]]} +// + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"47764370337599674\\\"]\\n,[1,\\\"9500405729277875\\\"]\\n,[2,\\\"57\\\"]\\n,[3,\\\"29330926862189\\\"]\\n,[4,\\\"909554854973\\\"]\\n]\\n}\",\r\n" + +// + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4\\\"]\\n,[1,\\\"12431-3145-756\\\"]\\n,[2,\\\"null\\\"]\\n,[3,\\\"\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n" + // nw2.handleRequest(request, null, null); + // backup + + // + " \"body\": + // \"{\\n\\\"data\\\":[\\n[0,\\\"454-55545-45675\\\"]\\n,[1,\\\"1243-131457-56\\\"]\\n,[2,\\\"584785473839\\\"]\\n,[3,\\\"32323232323232\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n" + // + " \"body\": + // \"{\\n\\\"data\\\":[\\n[0,\\\"9500405729277875\\\"]\\n,[1,\\\"9500405729277875\\\"]\\n,[2,\\\"49\\\"]\\n,[3,\\\"64310756511368\\\"]\\n,[4,\\\"294461560470\\\"]\\n]\\n}\",\r\n" + // + " \"body\": + // \"{\\n\\\"data\\\":[\\n[0,\\\"0\\\"]\\n,[1,\\\"null\\\"]\\n,[2,\\\"45\\\"]\\n,[3,\\\"32323232323232\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n" + // temp{"data":[[0,"9500405729277875"],[1,"9500405729277875"],[2,"49"],[3,"64310756511368"],[4,"294461560470"]]} + nw2.handleRequest2(request, null, null); + } + + public void handleRequest2(String inputStream, OutputStream outputStream, Context context) throws IOException { + // context.getLogger().log("Input: " + inputStream); + // String input = IOUtils.toString(inputStream, "UTF-8"); + + Map bqErrorMap = new HashMap(); + String encdata = ""; + int error_count = 0; + int statusCode = 200; + + String input = inputStream; + // Map encryptedErrorMapTotal = new HashMap(); + + JsonObject bodyObject = new JsonObject(); + JsonArray dataArray = new JsonArray(); + JsonArray innerDataArray = new JsonArray(); + String snowflakereturnstring = null; + JsonObject body_input_request = null; + + int numberofchunks = 0; + + String callerStr = null; + + // https://www.baeldung.com/java-aws-lambda + + JsonObject snowflakeinput = null; + + JsonArray snowflakedata = null; + + String keyName = "testfaas"; + String crdpip = System.getenv("CRDPIP"); + // String keyName = System.getenv("CMKEYNAME"); + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned + // when the user above does not have access to the key and if doing a + // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no + // then an error will be returned to the query, else the results set will provide ciphertext. + String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess"); + // yes,no + boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes"); + // usersetlookup = should a userset lookup be done on the user from Cloud DB + // yes,no + String usersetlookup = System.getenv("usersetlookup"); + // usersetidincm = should be the usersetid in CM to query. + String usersetID = System.getenv("usersetidincm"); + // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be + // a memcache or other in memory db. + String userSetLookupIP = System.getenv("usersetlookupip"); + boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes"); + String keymetadatalocation = System.getenv("keymetadatalocation"); + String external_version_from_ext_source = System.getenv("keymetadata"); + String protection_profile = System.getenv("protection_profile"); + String mode = System.getenv("mode"); + String datatype = System.getenv("datatype"); + // int batchsize = Integer.parseInt(System.getenv("BATCHSIZE")); + + String inputDataKey = null; + String outputDataKey = null; + String protectedData = null; + String externalkeymetadata = null; + String jsonBody = null; + String jsonTagForProtectReveal = null; + + boolean bad_data = false; + if (mode.equals("protectbulk")) { + inputDataKey = "data_array"; + outputDataKey = "protected_data_array"; + jsonTagForProtectReveal = PROTECTRETURNTAG; + } else { + inputDataKey = "protected_data_array"; + outputDataKey = "data_array"; + jsonTagForProtectReveal = REVEALRETURNTAG; + } + Response crdp_response = null; + + try { + + JsonElement rootNode = JsonParser.parseString(input).getAsJsonObject(); + if (rootNode.isJsonObject()) { + snowflakeinput = rootNode.getAsJsonObject(); + if (snowflakeinput.isJsonObject()) { + // For some reason when using snowflake it adds \n and \ to quotes in json. + // the JsonParser.parseString(input).getAsJsonObject(); is supposed to remove + // all of those + // characters but it does not do it for snowflake json. + JsonElement bodyele = snowflakeinput.get("body"); + String bodystr = bodyele.getAsString().replaceAll(System.lineSeparator(), ""); + bodystr = bodystr.replaceAll("\\\\", ""); + // System.out.println("bodystr after replace" + bodystr); + body_input_request = gson.fromJson(bodystr, JsonObject.class); + snowflakedata = body_input_request.getAsJsonArray("data"); + + JsonObject requestContext = snowflakeinput.getAsJsonObject("requestContext"); + + if (requestContext != null) { + JsonObject identity = requestContext.getAsJsonObject("identity"); + + if (identity != null) { + callerStr = identity.get("user").getAsString(); + System.out.println("user: " + callerStr); + } else { + System.out.println("Identity not found."); + } + } else { + System.out.println("Request context not found."); + } + + if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group + + boolean founduserinuserset = findUserInUserSet(callerStr, userName, password, usersetID, + userSetLookupIP); + // System.out.println("Found User " + founduserinuserset); + if (!founduserinuserset) + throw new CustomException("1001, User Not in User Set", 1001); + + } else { + usersetlookupbool = false; + } + + } else { + System.out.println("eerror"); + + } + } + + StringBuffer protection_policy_buff = new StringBuffer(); + String notvalid = "notvalid"; + + int numberOfLines = snowflakedata.size(); + int totalRowsLeft = numberOfLines; + + int batchsize = 3; + if (batchsize > numberOfLines) + batchsize = numberOfLines; + if (batchsize >= BATCHLIMIT) + batchsize = BATCHLIMIT; + // Serialization + + String crdpjsonBody = null; + // String external_version_from_ext_source = "1004001"; + // String external_version_from_ext_source = "1001001"; + + int i = 0; + int count = 0; + int totalcount = 0; + boolean newchunk = true; + int dataIndex = 0; // assumes index from snowflake will always be sequential. + JsonObject crdp_payload = new JsonObject(); + String sensitive = null; + JsonArray crdp_payload_array = new JsonArray(); + + OkHttpClient client = new OkHttpClient().newBuilder().build(); + MediaType mediaType = MediaType.parse("application/json"); + String urlStr = "http://" + crdpip + ":8090/v1/" + mode; + + while (i < numberOfLines) { + + for (int b = 0; b < batchsize && b < totalRowsLeft; b++) { + + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + sensitive = checkValid(snowflakerow); + protection_profile = protection_profile.trim(); + // Format the output + String formattedElement = String.format("\"protection_policy_name\" : \"%s\"", protection_profile); + protection_policy_buff.append(formattedElement); + protection_policy_buff.append(","); + + if (mode.equals("protectbulk")) { + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + // System.out.println("adding null not charint or nbr"); + sensitive = sensitive.replace("notvalid", ""); + sensitive = BADDATATAG + sensitive; + } else + sensitive = BADDATATAG; + + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + + } else if (sensitive.contains("notvalid")) { + // sensitive = sensitive.replace("notvalid", ""); + + } + encdata = sensitive; + + } + crdp_payload_array.add(sensitive); + } else { + JsonObject protectedDataObject = new JsonObject(); + protectedDataObject.addProperty("protected_data", sensitive); + if (keymetadatalocation.equalsIgnoreCase("external")) { + protectedDataObject.addProperty("external_version", external_version_from_ext_source); + } + crdp_payload_array.add(protectedDataObject); + + } + + if (count == batchsize - 1) { + crdp_payload.add(inputDataKey, crdp_payload_array); + String inputdataarray = null; + if (mode.equals("revealbulk")) { + crdp_payload.addProperty("username", callerStr); + inputdataarray = crdp_payload.toString(); + protection_policy_buff.append(inputdataarray); + jsonBody = protection_policy_buff.toString(); + jsonBody = jsonBody.replaceFirst("\\{", " "); + + } else { + inputdataarray = crdp_payload.toString(); + protection_policy_buff.append(inputdataarray); + inputdataarray = protection_policy_buff.toString(); + jsonBody = inputdataarray.replace("{", " "); + } + jsonBody = "{" + jsonBody; + + System.out.println(jsonBody); + RequestBody body = RequestBody.create(mediaType, jsonBody); + + Request crdp_request = new Request.Builder().url(urlStr).method("POST", body) + .addHeader("Content-Type", "application/json").build(); + crdp_response = client.newCall(crdp_request).execute(); + String crdpreturnstr = null; + if (crdp_response.isSuccessful()) { + // Parse JSON response + String responseBody = crdp_response.body().string(); + JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class); + JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey); + + String status = jsonObject.get("status").getAsString(); + int success_count = jsonObject.get("success_count").getAsInt(); + error_count = jsonObject.get("error_count").getAsInt(); + if (error_count > 0) + System.out.println("errors " + error_count); + + for (JsonElement element : protectedDataArray) { + + JsonObject protectedDataObject = element.getAsJsonObject(); + if (protectedDataObject.has(jsonTagForProtectReveal)) { + + protectedData = protectedDataObject.get(jsonTagForProtectReveal).getAsString(); + System.out.println(protectedData); + + innerDataArray.add(dataIndex); + innerDataArray.add(new String(protectedData)); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + if (mode.equals("protectbulk")) { + if (keymetadatalocation.equalsIgnoreCase("external") + && mode.equalsIgnoreCase("protect")) { + externalkeymetadata = protectedDataObject.get("external_version") + .getAsString(); + System.out.println("Protected Data ext key metadata need to store this: " + + externalkeymetadata); + + } + } + } else if (protectedDataObject.has("error_message")) { + String errorMessage = protectedDataObject.get("error_message").getAsString(); + System.out.println("error_message: " + errorMessage); + bqErrorMap.put(i, errorMessage); + bad_data = true; + } else + System.out.println("unexpected json value from results: "); + dataIndex++; + + } + + crdp_payload_array = new JsonArray(); + protection_policy_buff = new StringBuffer(); + numberofchunks++; + totalcount = totalcount + count; + count = 0; + + } else {// throw error.... + System.err.println("Request failed with status code: " + crdp_response.code()); + throw new CustomException("1010, Unexpected Error ", 1010); + } + + } else { + count++; + } + totalRowsLeft--; + i++; + } + } + if (count > 0) { + crdp_payload.add(inputDataKey, crdp_payload_array); + String inputdataarray = null; + if (mode.equals("revealbulk")) { + crdp_payload.addProperty("username", callerStr); + inputdataarray = crdp_payload.toString(); + protection_policy_buff.append(inputdataarray); + jsonBody = protection_policy_buff.toString(); + jsonBody = jsonBody.replaceFirst("\\{", " "); + + } else { + inputdataarray = crdp_payload.toString(); + protection_policy_buff.append(inputdataarray); + inputdataarray = protection_policy_buff.toString(); + jsonBody = inputdataarray.replace("{", " "); + } + jsonBody = "{" + jsonBody; + + RequestBody body = RequestBody.create(mediaType, jsonBody); + + Request crdp_request = new Request.Builder().url(urlStr).method("POST", body) + .addHeader("Content-Type", "application/json").build(); + crdp_response = client.newCall(crdp_request).execute(); + String crdpreturnstr = null; + if (crdp_response.isSuccessful()) { + // Parse JSON response + String responseBody = crdp_response.body().string(); + JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class); + JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey); + + String status = jsonObject.get("status").getAsString(); + int success_count = jsonObject.get("success_count").getAsInt(); + error_count = jsonObject.get("error_count").getAsInt(); + if (error_count > 0) + System.out.println("errors " + error_count); + + for (JsonElement element : protectedDataArray) { + + JsonObject protectedDataObject = element.getAsJsonObject(); + if (protectedDataObject.has(jsonTagForProtectReveal)) { + + protectedData = protectedDataObject.get(jsonTagForProtectReveal).getAsString(); + // System.out.println(protectedData); + + innerDataArray.add(dataIndex); + innerDataArray.add(new String(protectedData)); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + if (mode.equals("protectbulk")) { + if (keymetadatalocation.equalsIgnoreCase("external") + && mode.equalsIgnoreCase("protect")) { + externalkeymetadata = protectedDataObject.get("external_version").getAsString(); + // System.out.println("Protected Data ext key metadata need to store this: " + // + externalkeymetadata); + + } + } + } else if (protectedDataObject.has("error_message")) { + String errorMessage = protectedDataObject.get("error_message").getAsString(); + System.out.println("error_message: " + errorMessage); + bqErrorMap.put(i, errorMessage); + bad_data = true; + } else + System.out.println("unexpected json value from results: "); + dataIndex++; + + } + + crdp_payload_array = new JsonArray(); + protection_policy_buff = new StringBuffer(); + numberofchunks++; + totalcount = totalcount + count; + count = 0; + + } else { + System.err.println("Request failed with status code: " + crdp_response.code()); + } + } + + crdp_response.close(); + System.out.println("total chuncks " + numberofchunks); + + bodyObject.add("data", dataArray); + JsonObject inputJsonObject = new JsonObject(); + String bodyString = bodyObject.toString(); + inputJsonObject.addProperty("statusCode", 200); + inputJsonObject.addProperty("body", bodyString); + + snowflakereturnstring = inputJsonObject.toString(); + + } catch (Exception e) { + System.out.println("in exception with " + e.getMessage()); + snowflakereturnstring = "exception "; + if (returnciphertextbool) { + if (e.getMessage().contains("1401") + || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) { + + bodyObject = new JsonObject(); + dataArray = new JsonArray(); + innerDataArray = new JsonArray(); + + for (int i = 0; i < snowflakedata.size(); i++) { + + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + for (int j = 0; j < snowflakerow.size(); j++) { + if (j == 1) { + // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); + // FPE example + String sensitive = checkValid(snowflakerow); + + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + } else + sensitive = BADDATATAG; + + } else if (sensitive.equalsIgnoreCase("null") + || sensitive.equalsIgnoreCase("notvalid")) { + + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + + } + encdata = sensitive; + + } else { + System.out.println("normal number data" + sensitive); + } + innerDataArray.add(sensitive); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + int row_number = snowflakecolumn.getAsInt(); + innerDataArray.add(row_number); + } + } + } + + bodyObject.add("data", dataArray); + JsonObject inputJsonObject = new JsonObject(); + String bodyString = bodyObject.toString(); + inputJsonObject.addProperty("statusCode", 200); + inputJsonObject.addProperty("body", bodyString); + + snowflakereturnstring = inputJsonObject.toString(); + System.out.println(" new data " + snowflakereturnstring); + + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + + } finally { + + } + System.out.println(snowflakereturnstring); + // response.getWriter().write(snowflakereturnstring); + + } + + public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID, + String userSetLookupIP) throws Exception { + + CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP); + + String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd); + String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken); + + boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken); + + return founduserinuserset; + + } + + public String checkValid(JsonArray snowrow) { + String inputdata = null; + String notvalid = "notvalid"; + if (snowrow != null && snowrow.size() > 0) { + JsonElement element = snowrow.get(1); + if (element != null && !element.isJsonNull()) { + inputdata = element.getAsString(); + if (inputdata.isEmpty() || inputdata.length() < 2) { + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("Sensitive data is null or empty."); + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("bigquerytrow is null or empty."); + inputdata = notvalid + inputdata; + } + + return inputdata; + + } + + public String formatReturnValue(int statusCode) + + { + StringBuffer snowflakereturndatasb = new StringBuffer(); + + snowflakereturndatasb.append("{ \"statusCode\":"); + snowflakereturndatasb.append(statusCode); + snowflakereturndatasb.append(","); + snowflakereturndatasb.append(" \"body\": {"); + snowflakereturndatasb.append(" \"data\": ["); + snowflakereturndatasb.append("] }}"); + System.out.println("in exception with "); + return snowflakereturndatasb.toString(); + } + + @Override + public void handleRequest(InputStream input, OutputStream output, Context context) throws IOException { + // TODO Auto-generated method stub + + } + +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCRDPFPEUDFTester.java b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCRDPFPEUDFTester.java new file mode 100644 index 00000000..fa8da656 --- /dev/null +++ b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCRDPFPEUDFTester.java @@ -0,0 +1,515 @@ + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.HashMap; +import java.util.Map; + +import com.amazonaws.services.lambda.runtime.Context; +import com.amazonaws.services.lambda.runtime.RequestStreamHandler; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; + +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +/* This sample AWS Lambda Function is used to implement a Snowflake Database User Defined Function(UDF). /* + * It is an example of how to use Thales Cipher Trust Application Data Protection (CADP) + * to protect sensitive data in a column. This example uses + * Format Preserve Encryption (FPE) to maintain the original format of the data + * so applications or business intelligence tools do not have to change in order + * to use these columns. + * +* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested +* for all possible data sizes and combinations of encryption algorithms and IV, etc. +* Was tested with CM 2.14 & CADP CADP 8.15.0.001 +* For more information on CADP see link below. +https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html +* For more information on Snowflake External Functions see link below. +https://docs.snowflake.com/en/sql-reference/external-functions-creating-aws + * + */ + +public class ThalesAWSSnowCRDPFPEUDFTester implements RequestStreamHandler { + + private static final Gson gson = new Gson(); + private static final String BADDATATAG = new String("9999999999999999"); + private static final String REVEALRETURNTAG = new String("data"); + private static final String PROTECTRETURNTAG = new String("protected_data"); + + public static void main(String[] args) throws Exception + + { + ThalesAWSSnowCRDPFPEUDFTester nw2 = new ThalesAWSSnowCRDPFPEUDFTester(); + String request = "{\r\n" + " \"resource\": \"/snowflake_proxy\",\r\n" + " \"path\": \"/snowflake_proxy\",\r\n" + + " \"httpMethod\": \"POST\",\r\n" + " \"headers\": {\r\n" + " \"Accept\": \"*/*\",\r\n" + + " \"Accept-Encoding\": \"gzip\",\r\n" + " \"Content-Encoding\": \"gzip\",\r\n" + + " \"Content-Type\": \"application/json\",\r\n" + + " \"Host\": \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\",\r\n" + + " \"sf-external-function-current-query-id\": \"01b02bdf-0001-7b58-0003-27d60056b5a6\",\r\n" + + " \"sf-external-function-format\": \"json\",\r\n" + + " \"sf-external-function-format-version\": \"1.0\",\r\n" + + " \"sf-external-function-name\": \"thales-aws-lambda-snow-cadp-encrypt-nbr\",\r\n" + + " \"sf-external-function-name-base64\": \"VEhBTEVTX0NBRFBfQVdTX1RPS0VOSVpFQ0hBUg==\",\r\n" + + " \"sf-external-function-query-batch-id\": \"01b02bdf-0001-7b58-0003-27d60056b5a6:1:1:0:0\",\r\n" + + " \"sf-external-function-return-type\": \"VARIANT\",\r\n" + + " \"sf-external-function-return-type-base64\": \"VkFSSUFOVA==\",\r\n" + + " \"sf-external-function-signature\": \"(B VARCHAR)\",\r\n" + + " \"sf-external-function-signature-base64\": \"KEIgVkFSQ0hBUik=\",\r\n" + + " \"User-Agent\": \"snowflake/1.0\",\r\n" + + " \"x-amz-content-sha256\": \"550cead700c849cd6f159aca9dd81083306bb5c919939c1e2c4f3f909f32332b\",\r\n" + + " \"x-amz-date\": \"20231107T142353Z\",\r\n" + + " \"x-amz-security-token\": \"IQoJb3JpZ2luX2VjEG8aCXVzLWVhc3QtMSJGMEQCICKJkvz9H9bTmSg3rRMNQYHnGeNzDyseSK3GxkGkxH6NAiBm/1kqxf5r25Z7154EYa93MaMOFIdMKS/b7MheulrJxCqHAwio//////////8BEAAaDDQ5NjQ4NDM0Mjc2NCIMCJwVrXJkl8t82Se4KtsC1mBjS4CLPcQEHJ+qRnw5HHbMTWx1DkJ6SeAjMWj/QypGy3yipoi20gY49xkbo4A0kzv7bty06Oa/VPp4id/hWdVEPVypuUQpyHKYdJAn/ZrnhvyqOMlKpGSg3157v1zPduOQOz9XvjpWC12NCaF3HAcj2/aH0DfCRacsAS+XwIWxNFOtrGbsbibrSJqFe2NqbvIbLU39cVLCD51cTMRMTXhdcRpU9JhXc8+I8+eWwsgpl823OrjwTwqDCnYs/9AY+yRCUDVjuLbKKQ+lXNWlaOMD6YfBzgNnjZZDbctx1lSMHG7HRHQrFIuZe0oKnVkbeF05CSxLBorfigKX8AF/Kyq5OhSnJPdTZKG6PbtWAub+JHaelZdEo3wDTMN04MkL+Nef+gV+g2vxejZiOc+v0HL0qXB1z2OLDqXfMm1zoACue3sYLqeK2jRjWqpEoPes7rU7jHOgkrWDgfkw7ZCpqgY6ngGFyKE++chqS6vGrZJ2e/HQL12UVqyxfT2NDRQZDA0gZmWnK5drpNJECXH9DeFcVqUnnDU77Ui5v6tXfUrq9cSOOYG68b2qCYiQ+TNkQPiMZNJmx4FYDL46RWtcxyvAMqaQz/5bhwf9W/Iw2QzYuSWj4FTC29H0eD1B5Frnq69Eg3A1A1bJ8m0LfDhT7gfK/rp7I2N0LSnYbhi2ksZU6A==\",\r\n" + + " \"X-Amzn-Trace-Id\": \"Root=1-654a4879-4aa55d5e4a5648a0268543d1\",\r\n" + + " \"X-Forwarded-For\": \"3.33.33.35\",\r\n" + " \"X-Forwarded-Port\": \"443\",\r\n" + + " \"X-Forwarded-Proto\": \"https\"\r\n" + " },\r\n" + " \"multiValueHeaders\": {\r\n" + + " \"Accept\": [\r\n" + " \"*/*\"\r\n" + " ],\r\n" + " \"Accept-Encoding\": [\r\n" + + " \"gzip\"\r\n" + " ],\r\n" + " \"Content-Encoding\": [\r\n" + " \"gzip\"\r\n" + + " ],\r\n" + " \"Content-Type\": [\r\n" + " \"application/json\"\r\n" + " ],\r\n" + + " \"Host\": [\r\n" + " \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\"\r\n" + " ],\r\n" + + " \"sf-external-function-current-query-id\": [\r\n" + " \"sdfsdfsdff-27d60056b5a6\"\r\n" + + " ],\r\n" + " \"sf-external-function-format\": [\r\n" + " \"json\"\r\n" + " ],\r\n" + + " \"sf-external-function-format-version\": [\r\n" + " \"1.0\"\r\n" + " ],\r\n" + + " \"sf-external-function-name\": [\r\n" + " \"THALES_CADP_AWS_R\"\r\n" + " ],\r\n" + + " \"sf-external-function-name-base64\": [\r\n" + " \"wewewewewewe==\"\r\n" + " ],\r\n" + + " \"sf-external-function-query-batch-id\": [\r\n" + + " \"01b02bdf-0001-7b58-0003-27d60056b5a6:1:1:0:0\"\r\n" + " ],\r\n" + + " \"sf-external-function-return-type\": [\r\n" + " \"VARIANT\"\r\n" + " ],\r\n" + + " \"sf-external-function-return-type-base64\": [\r\n" + " \"VkFSSUFOVA==\"\r\n" + " ],\r\n" + + " \"sf-external-function-signature\": [\r\n" + " \"(B VARCHAR)\"\r\n" + " ],\r\n" + + " \"sf-external-function-signature-base64\": [\r\n" + " \"KEIgVkFSQ0hBUik=\"\r\n" + + " ],\r\n" + " \"User-Agent\": [\r\n" + " \"snowflake/1.0\"\r\n" + " ],\r\n" + + " \"x-amz-content-sha256\": [\r\n" + " \"550cead70rtrtrtrtb5c919939c1e2c4f3f909f32332b\"\r\n" + + " ],\r\n" + " \"x-amz-date\": [\r\n" + " \"20231107T142353Z\"\r\n" + " ],\r\n" + + " \"x-amz-security-token\": [\r\n" + + " \"IQoJb3JpZ2luX2VjEG8aCXVzLWVhc3QtMSJGMEQCICKJkvz9H9bTmSg3rRMNQYHnGeNzDyseSK3GxkGkxH6NAiBm/1kqxf5r25Z7154EYa93MaMOFIdMKS/b7MheulrJxCqHAwio//////////8BEAAaDDQ5NjQ4NDM0Mjc2NCIMCJwVrXJkl8t82Se4KtsC1mBjS4CLPcQEHJ+qRnw5HHbMTWx1DkJ6SeAjMWj/QypGy3yipoi20gY49xkbo4A0kzv7bty06Oa/VPp4id/hWdVEPVypuUQpyHKYdJAn/ZrnhvyqOMlKpGSg3157v1zPduOQOz9XvjpWC12NCaF3HAcj2/aH0DfCRacsAS+XwIWxNFOtrGbsbibrSJqFe2NqbvIbLU39cVLCD51cTMRMTXhdcRpU9JhXc8+I8+eWwsgpl823OrjwTwqDCnYs/9AY+yRCUDVjuLbKKQ+lXNWlaOMD6YfBzgNnjZZDbctx1lSMHG7HRHQrFIuZe0oKnVkbeF05CSxLBorfigKX8AF/Kyq5OhSnJPdTZKG6PbtWAub+JHaelZdEo3wDTMN04MkL+Nef+gV+g2vxejZiOc+v0HL0qXB1z2OLDqXfMm1zoACue3sYLqeK2jRjWqpEoPes7rU7jHOgkrWDgfkw7ZCpqgY6ngGFyKE++chqS6vGrZJ2e/HQL12UVqyxfT2NDRQZDA0gZmWnK5drpNJECXH9DeFcVqUnnDU77Ui5v6tXfUrq9cSOOYG68b2qCYiQ+TNkQPiMZNJmx4FYDL46RWtcxyvAMqaQz/5bhwf9W/Iw2QzYuSWj4FTC29H0eD1B5Frnq69Eg3A1A1bJ8m0LfDhT7gfK/rp7I2N0LSnYbhi2ksZU6A==\"\r\n" + + " ],\r\n" + " \"X-Amzn-Trace-Id\": [\r\n" + + " \"Root=1-654a4879-4aa55d5e4a5648a0268543d1\"\r\n" + " ],\r\n" + + " \"X-Forwarded-For\": [\r\n" + " \"3.34.44.35\"\r\n" + " ],\r\n" + + " \"X-Forwarded-Port\": [\r\n" + " \"443\"\r\n" + " ],\r\n" + + " \"X-Forwarded-Proto\": [\r\n" + " \"https\"\r\n" + " ]\r\n" + " },\r\n" + + " \"queryStringParameters\": null,\r\n" + " \"multiValueQueryStringParameters\": null,\r\n" + + " \"pathParameters\": null,\r\n" + " \"stageVariables\": null,\r\n" + " \"requestContext\": {\r\n" + + " \"resourceId\": \"erw8lc\",\r\n" + " \"resourcePath\": \"/snowflake_proxy\",\r\n" + + " \"httpMethod\": \"POST\",\r\n" + " \"extendedRequestId\": \"OCBC9FLtiYcEThQ=\",\r\n" + + " \"requestTime\": \"07/Nov/2023:14:23:53 +0000\",\r\n" + + " \"path\": \"/test/snowflake_proxy\",\r\n" + " \"accountId\": \"455555555564\",\r\n" + + " \"protocol\": \"HTTP/1.1\",\r\n" + " \"stage\": \"test\",\r\n" + + " \"domainPrefix\": \"asdfsdfsfff\",\r\n" + " \"requestTimeEpoch\": 1699367033023,\r\n" + + " \"requestId\": \"f53460af-d302-4e3a-b837-f5a9785badcb\",\r\n" + " \"identity\": {\r\n" + + " \"cognitoIdentityPoolId\": null,\r\n" + " \"accountId\": \"34343434\",\r\n" + + " \"cognitoIdentityId\": null,\r\n" + " \"caller\": \"ARODFDFDFFDFDFX7G:snowflake\",\r\n" + + " \"sourceIp\": \"3.33.44.35\",\r\n" + " \"principalOrgId\": null,\r\n" + + " \"accessKey\": \"FAKE\",\r\n" + " \"cognitoAuthenticationType\": null,\r\n" + + " \"cognitoAuthenticationProvider\": null,\r\n" + + " \"userArn\": \"arn:aws:sts::496484342764:assumed-role/snowflakerole/snowflake\",\r\n" + + " \"userAgent\": \"snowflake/1.0\",\r\n" + + " \"user\": \"shawnscanlan@snowflakecomputing.com\"\r\n" + " },\r\n" + + " \"domainName\": \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\",\r\n" + + " \"apiId\": \"asdfsdfsfff\"\r\n" + " },\r\n" + + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"789776\\\"]\\n]\\n}\",\r\n" + " \"isBase64Encoded\": false\r\n" + + "}"; + + // + " \"body\": + // \"{\\n\\\"data\\\":[\\n[0,\\\"4545554545675\\\"]\\n,[1,\\\"124313145756\\\"]\\n,[2,\\\"584785473839\\\"]\\n,[3,\\\"32323232323232\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n" + // + " \"body\": + // \"{\\n\\\"data\\\":[\\n[0,\\\"47764370337599674\\\"]\\n,[1,\\\"9500405729277875\\\"]\\n,[2,\\\"57\\\"]\\n,[3,\\\"64310756511368\\\"]\\n,[4,\\\"909554854973\\\"]\\n]\\n}\",\r\n" + // + " \"body\": + // \"{\\n\\\"data\\\":[\\n[0,\\\"6\\\"]\\n,[1,\\\"null\\\"]\\n,[2,\\\"49\\\"]\\n,[3,\\\"64310756511368\\\"]\\n,[4,\\\"294461560470\\\"]\\n]\\n}\",\r\n" + // temp{"data":[[0,"1676960812748"],[1,"933211143272"],[2,"505141998387"],[3,"64310756511368"],[4,"294461560470"]]} + // reveal 1002001 + // temp{"data":[[0,"47764370337599674"],[1,"9500405729277875"],[2,"57"],[3,"29330926862189"],[4,"909554854973"]]} +// + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"47764370337599674\\\"]\\n,[1,\\\"9500405729277875\\\"]\\n,[2,\\\"57\\\"]\\n,[3,\\\"29330926862189\\\"]\\n,[4,\\\"909554854973\\\"]\\n]\\n}\",\r\n" + +// + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4\\\"]\\n,[1,\\\"12431-3145-756\\\"]\\n,[2,\\\"null\\\"]\\n,[3,\\\"\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n" + + // + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4545554545675\\\"]\\n]\\n}\",\r\n" +// + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4545554-5456-75\\\"]\\n]\\n}\",\r\n" + // "body": + // "{\n\"data\":[\n[0,\"ndorgan5@sf_tuts.com\"],\n[1,\"cdevereu6@sf_tuts.co.au\"],\n[2,\"gglaserman7@sf_tuts.com\"],\n[3,\"icasemore8@sf_tuts.com\"],\n[4,\"chovie9@sf_tuts.com\"],\n[5,\"afeatherstona@sf_tuts.com\"],\n[6,\"hanneslieb@sf_tuts.com\"],\n[7,\"bciccoc@sf_tuts.com\"],\n[8,\"bdurnalld@sf_tuts.com\"],\n[9,\"kmacconnale@sf_tuts.com\"],\n[10,\"wsizeyf@sf_tuts.com\"],\n[11,\"dmcgowrang@sf_tuts.com\"],\n[12,\"cbedderh@sf_tuts.co.au\"],\n[13,\"davoryi@sf_tuts.com\"],\n[14,\"rtalmadgej@sf_tuts.co.uk\"],\n[15,\"lboissier@sf_tuts.com\"],\n[16,\"ihanks1@sf_tuts.com\"],\n[17,\"alaudham2@sf_tuts.com\"],\n[18,\"ecornner3@sf_tuts.com\"],\n[19,\"hgoolding4@sf_tuts.com\"],\n[20,\"adavidovitsk@sf_tuts.com\"],\n[21,\"vshermorel@sf_tuts.com\"],\n[22,\"rmattysm@sf_tuts.com\"],\n[23,\"soluwatoyinn@sf_tuts.com\"],\n[24,\"gbassfordo@sf_tuts.co.uk\"]\n]\n}", + // "isBase64Encoded": false + + // " \"user\": \"ADFDFDFFG:snowflake\"\r\n" + + // + " \"user\": \"shawnscanlan@snowflakecomputing.com\"\r\n" + " },\r\n" + nw2.handleRequest(request, null, null); + + } + + public void handleRequest(String inputStream, OutputStream outputStream, Context context) throws Exception { + Map bqErrorMap = new HashMap(); + String encdata = ""; + String snowflakereturnstring = null; + + String input = inputStream; + JsonObject snowflakebody = null; + int statusCode = 200; + + // https://www.baeldung.com/java-aws-lambda + JsonObject snowflakeinput = null; + String callerStr = null; + JsonArray snowflakedata = null; + + String keyName = "testfaas"; + String crdpip = System.getenv("CRDPIP"); + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be + // returned when the user above does not have access to the key and if doing a lookup in the userset + // and the user does not exist. If returnciphertextforuserwithnokeyaccess = no then an error will be + // returned to the query, else the results set will provide ciphertext. + // yes/no + String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess"); + boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes"); + + // usersetlookup = should a userset lookup be done on the user from Cloud DB? + // yes/no + String usersetlookup = System.getenv("usersetlookup"); + // usersetidincm = should be the usersetid in CM to query. + String usersetID = System.getenv("usersetidincm"); + // usersetlookupip = this is the IP address to query the userset. Currently it + // is the userset in CM but could be a memcache or other in memory db. + String userSetLookupIP = System.getenv("usersetlookupip"); + boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes"); + // keymetadatalocation keymeta data can be internal or external + String keymetadatalocation = System.getenv("keymetadatalocation"); + // keymetadata represents a 7 digit value that contains policy version and key version. example 1001001. + // Normally would come from a database + String external_version_from_ext_source = System.getenv("keymetadata"); + // protection_profile = the protection profile to be used for protect or reveal. This is in CM under application + // data protection/protection profiles. + String protection_profile = System.getenv("protection_profile"); + // mode of operation. valid values are protect/reveal + String mode = System.getenv("mode"); + String datatype = System.getenv("datatype"); + String dataKey = null; + JsonObject bodyObject = new JsonObject(); + JsonArray dataArray = new JsonArray(); + JsonArray innerDataArray = new JsonArray(); + String jsonTagForProtectReveal = null; + + boolean bad_data = false; + if (mode.equals("protect")) { + dataKey = "data"; + jsonTagForProtectReveal = PROTECTRETURNTAG; + } else { + dataKey = "protected_data"; + jsonTagForProtectReveal = REVEALRETURNTAG; + } + + // This code is only to be used when input data contains user info. + + try { + + JsonElement rootNode = JsonParser.parseString(input).getAsJsonObject(); + if (rootNode.isJsonObject()) { + snowflakeinput = rootNode.getAsJsonObject(); + if (snowflakeinput.isJsonObject()) { + // For some reason when using snowflake it adds \n and \ to quotes in json. + // the JsonParser.parseString(input).getAsJsonObject(); is supposed to remove + // all of those + // characters but it does not do it for snowflake json. + JsonElement bodyele = snowflakeinput.get("body"); + String bodystr = bodyele.getAsString().replaceAll(System.lineSeparator(), ""); + // System.out.println("bodystr before replace" + bodystr ); + bodystr = bodystr.replaceAll("\\\\", ""); + // System.out.println("bodystr after replace" + bodystr ); + snowflakebody = gson.fromJson(bodystr, JsonObject.class); + snowflakedata = snowflakebody.getAsJsonArray("data"); + JsonObject requestContext = snowflakeinput.getAsJsonObject("requestContext"); + + if (requestContext != null) { + JsonObject identity = requestContext.getAsJsonObject("identity"); + + if (identity != null) { + callerStr = identity.get("user").getAsString(); + System.out.println("user: " + callerStr); + } else { + System.out.println("Identity not found."); + } + } else { + System.out.println("Request context not found."); + } + + if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group + + boolean founduserinuserset = findUserInUserSet(callerStr, userName, password, usersetID, + userSetLookupIP); + // System.out.println("Found User " + founduserinuserset); + if (!founduserinuserset) + throw new CustomException("1001, User Not in User Set", 1001); + + } else { + usersetlookupbool = false; + } + + } else { + System.out.println("eerror"); + + } + } + + // Serialization + + String protectedData = null; + String externalkeymetadata = null; + String crdpjsonBody = null; + // String external_version_from_ext_source = "1004001"; + // String external_version_from_ext_source = "1001001"; + int row_number = 0; + JsonObject crdp_payload = new JsonObject(); + + crdp_payload.addProperty("protection_policy_name", protection_profile); + String sensitive = null; + OkHttpClient client = new OkHttpClient().newBuilder().build(); + MediaType mediaType = MediaType.parse("application/json"); + String urlStr = "http://" + crdpip + ":8090/v1/" + mode; + + + for (int i = 0; i < snowflakedata.size(); i++) { + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + for (int j = 0; j < snowflakerow.size(); j++) { + + if (j == 1) { + // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); + // FPE example + sensitive = checkValid(snowflakerow); + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + } else + sensitive = BADDATATAG; + + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + + } + encdata = sensitive; + + } else { + + crdp_payload.addProperty(dataKey, sensitive); + if (mode.equals("reveal")) { + crdp_payload.addProperty("username", callerStr); + if (keymetadatalocation.equalsIgnoreCase("external")) { + crdp_payload.addProperty("external_version", external_version_from_ext_source); + } + } + crdpjsonBody = crdp_payload.toString(); + System.out.println(crdpjsonBody); + RequestBody body = RequestBody.create(mediaType, crdpjsonBody); + + // String urlStr = "\"http://" + cmip + ":8090/v1/" + mode+ "\""; + System.out.println(urlStr); + Request crdp_request = new Request.Builder() + // .url("http://192.168.159.143:8090/v1/protect").method("POST", body) + .url(urlStr).method("POST", body).addHeader("Content-Type", "application/json") + .build(); + Response crdp_response = client.newCall(crdp_request).execute(); + + if (crdp_response.isSuccessful()) { + + // Parse JSON response + String responseBody = crdp_response.body().string(); + Gson gson = new Gson(); + JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class); + + if (jsonObject.has(jsonTagForProtectReveal)) { + protectedData = jsonObject.get(jsonTagForProtectReveal).getAsString(); + if (keymetadatalocation.equalsIgnoreCase("external")) { + externalkeymetadata = jsonObject.get("external_version").getAsString(); + System.out.println("Protected Data ext key metadata need to store this: " + + externalkeymetadata); + } + } else if (jsonObject.has("error_message")) { + String errorMessage = jsonObject.get("error_message").getAsString(); + System.out.println("error_message: " + errorMessage); + bqErrorMap.put(i, errorMessage); + bad_data = true; + } else + System.out.println("unexpected json value from results: "); + } else { + System.err.println("Request failed with status code: " + crdp_response.code()); + } + + crdp_response.close(); + + encdata = protectedData; + + } + + innerDataArray.add(encdata); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + row_number = snowflakecolumn.getAsInt(); + innerDataArray.add(row_number); + } + + } + + } + + bodyObject.add("data", dataArray); + JsonObject inputJsonObject = new JsonObject(); + String bodyString = bodyObject.toString(); + inputJsonObject.addProperty("statusCode", 200); + inputJsonObject.addProperty("body", bodyString); + + snowflakereturnstring = inputJsonObject.toString(); + System.out.println(" new data " + snowflakereturnstring); + + } catch (Exception e) { + + System.out.println("in exception with " + e.getMessage()); + snowflakereturnstring = "exception "; + if (returnciphertextbool) { + if (e.getMessage().contains("1401") + || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) { + + bodyObject = new JsonObject(); + dataArray = new JsonArray(); + innerDataArray = new JsonArray(); + + for (int i = 0; i < snowflakedata.size(); i++) { + + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + for (int j = 0; j < snowflakerow.size(); j++) { + if (j == 1) { + // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); + // FPE example + String sensitive = checkValid(snowflakerow); + + sensitive = checkValid(snowflakerow); + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + System.out.println("adding null not charint or nbr"); + sensitive = sensitive.replace("notvalid", ""); + } else + sensitive = BADDATATAG; + + } else if (sensitive.equalsIgnoreCase("null") + || sensitive.equalsIgnoreCase("notvalid")) { + + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + + } + encdata = sensitive; + + } else { + System.out.println("normal number data" + sensitive); + } + innerDataArray.add(sensitive); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + int row_number = snowflakecolumn.getAsInt(); + innerDataArray.add(row_number); + } + } + } + + bodyObject.add("data", dataArray); + JsonObject inputJsonObject = new JsonObject(); + String bodyString = bodyObject.toString(); + inputJsonObject.addProperty("statusCode", 200); + inputJsonObject.addProperty("body", bodyString); + + snowflakereturnstring = inputJsonObject.toString(); + System.out.println(" new data " + snowflakereturnstring); + + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + + } finally { + + } + System.out.println("results" + snowflakereturnstring); + // outputStream.write(snowflakereturnstring.getBytes()); + + } + + public String formatReturnValue(int statusCode) + + { + StringBuffer snowflakereturndatasb = new StringBuffer(); + + snowflakereturndatasb.append("{ \"statusCode\":"); + snowflakereturndatasb.append(statusCode); + snowflakereturndatasb.append(","); + snowflakereturndatasb.append(" \"body\": {"); + snowflakereturndatasb.append(" \"data\": ["); + snowflakereturndatasb.append("] }}"); + System.out.println("in exception with "); + return snowflakereturndatasb.toString(); + } + + public String checkValid(JsonArray snowrow) { + String inputdata = null; + String notvalid = "notvalid"; + if (snowrow != null && snowrow.size() > 0) { + JsonElement element = snowrow.get(1); + if (element != null && !element.isJsonNull()) { + inputdata = element.getAsString(); + if (inputdata.isEmpty() || inputdata.length() < 2) { + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("Sensitive data is null or empty."); + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("bigquerytrow is null or empty."); + inputdata = notvalid + inputdata; + } + + return inputdata; + + } + + @Override + public void handleRequest(InputStream input, OutputStream output, Context context) throws IOException { + // TODO Auto-generated method stub + + } + + public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID, + String userSetLookupIP) throws Exception { + + CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP); + + String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd); + String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken); + + boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken); + + return founduserinuserset; + + } + +} \ No newline at end of file diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/pom.xml b/database/snowflake/Thales-Snow-GCP-UDF/pom.xml similarity index 94% rename from database/snowflake/CADP-SNOW-GCP-Functions/pom.xml rename to database/snowflake/Thales-Snow-GCP-UDF/pom.xml index 564bed67..578df43b 100644 --- a/database/snowflake/CADP-SNOW-GCP-Functions/pom.xml +++ b/database/snowflake/Thales-Snow-GCP-UDF/pom.xml @@ -4,8 +4,8 @@ 4.0.0 Thales - CADP-SNOW-GCP-UDF - 0.0.2-SNAPSHOT + Thales-Snow-GCP-UDF + 0.0.9-SNAPSHOT 11 11 @@ -20,7 +20,7 @@ 2.17.2 .000 - CADP-SNOW-GCP-UDF + Thales-Snow-GCP-UDF io.github.thalescpl-io.cadp @@ -103,11 +103,6 @@ log4j-core ${log4j-core.version} - - IngrianNAE - IngrianNAE - 8.12.6.000 - diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/CMUserSetHelper.java b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/CMUserSetHelper.java new file mode 100644 index 00000000..edd5b198 --- /dev/null +++ b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/CMUserSetHelper.java @@ -0,0 +1,508 @@ +package com.example; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; + +import java.io.BufferedReader; +import java.io.DataOutputStream; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.security.KeyStore; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; +/* + * This app provides a number of different helper methods dealing with CM Application Data Protection UserSets. There is a method to find + * a user in a userset and another method to populate the userset from a flat file. Usersets are typically used within + * an access policy but they are not restricted to that usage. + * Was tested with CM 2.14 + * Note: This source code is only to be used for testing and proof of concepts. + * @author mwarner + * + */ + +public class CMUserSetHelper { + + static String hostnamevalidate = "yourhostname"; + + String usersetid = "716f01a6-5cab-4799-925a-6dc2d8712fc1"; + String cmIP = "yourip"; + String apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?name="; + // + username_in_userset; + String addusertouserset = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users"; + + int totalrecords = 0; + String authUrl = "https://" + cmIP + "/api/v1/auth/tokens"; + + static boolean debug = true; + int chunksize = 5; + static int CHUNKSIZEMAX = 100; + + public CMUserSetHelper(String usersetid, String cmIP) { + + this.usersetid = usersetid; + this.cmIP = cmIP; + this.apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?name="; + // this.apiUrlGetUsers = "https://" + cmIP + + // "/api/v1/data-protection/user-sets/" + usersetid + "/users?limit=" + + // this.usersetlimit + "&name="; + this.addusertouserset = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users"; + this.authUrl = "https://" + cmIP + "/api/v1/auth/tokens"; + } + + public static void main(String[] args) throws Exception { + + String username = args[0]; + String password = args[1]; + String cmip = args[2]; + String usersetid = args[3]; + String filePath = args[4]; + + CMUserSetHelper cmusersetHelper = new CMUserSetHelper(usersetid, cmip); + int totalrecords = 0; + + // String apiUrl = apiUrlGetUsers; + String jwthtoken = geAuthToken(cmusersetHelper.authUrl, username, password); + + String newtoken = "Bearer " + removeQuotes(jwthtoken); + + // String filePath = + // "C:\\Users\\t0185905\\workspace\\CT-VL-GCP\\src\\main\\java\\com\\example\\emailAddresses.txt"; + RandomAccessFile file = new RandomAccessFile(filePath, "r"); + if (cmusersetHelper.chunksize > CHUNKSIZEMAX) + cmusersetHelper.chunksize = CHUNKSIZEMAX; + int totoalnbrofrecords = numberOfLines(file); + if (cmusersetHelper.chunksize > totoalnbrofrecords) { + cmusersetHelper.chunksize = totoalnbrofrecords / 2; + } + totalrecords = cmusersetHelper.addAUserToUserSetFromFile(cmusersetHelper.addusertouserset, newtoken, filePath); + System.out.println("Totalrecords inserted into Userset " + cmusersetHelper.usersetid + " = " + totalrecords); + + } + + /** + * Returns an boolean if user found + *

+ * + * @param user user to find + * @param newtoken jwt token to use + * @return boolean true if found in userset + * @throws CustomException + */ + public boolean findUserInUserSet(String user, String newtoken) throws CustomException { + boolean found = false; + + String apiUrl = this.apiUrlGetUsers + user; + + apiUrl = removeQuotes(apiUrl); + + try { + URL url = new URL(apiUrl); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("GET"); + connection.setRequestProperty("Authorization", newtoken); + connection.setRequestProperty("accept", "application/json"); + + connection.setDoInput(true); + BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + + StringBuilder response = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + response.append(line); + } + reader.close(); + + Gson gson = new Gson(); + if (debug) + System.out.println("response " + response); + JsonObject input = null; + JsonElement total = null; + JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject(); + if (rootNode.isJsonObject()) { + input = rootNode.getAsJsonObject(); + if (input.isJsonObject()) { + total = input.get("total"); + } + } + JsonPrimitive column = total.getAsJsonPrimitive(); + String totalstr = column.getAsJsonPrimitive().toString(); + + Integer i = Integer.valueOf(totalstr); + + if (i > 0) { + found = true; + + } + connection.disconnect(); + } catch (Exception e) { + if (e.getMessage().contains("403")) { + throw new CustomException("1002, User Not in Application Data Protection Clients ", 1002); + } else + e.printStackTrace(); + } + + return found; + + } + + /** + * Loads users from a file to the userset. Returns an int of number of users + * added. + *

+ * + * @param url url to userset api + * @param newtoken jwt token to use + * @param filePath file to load + * @return int totalnumberofrecords added to userset + */ + + public int addAUserToUserSetFromFile(String url, String newtoken, String filePath) throws IOException { + + int totalnbrofrecords = 0; + try (BufferedReader br = new BufferedReader(new FileReader(filePath))) { + String line; + int count = 0; + StringBuilder payloadBuilder = new StringBuilder(); + + payloadBuilder.append("{\"users\": ["); + + while ((line = br.readLine()) != null) { + totalnbrofrecords++; + // Append the email address to the payload + payloadBuilder.append("\"").append(line).append("\","); + count++; + + // If 'n' records have been read, print the payload + if (count == chunksize) { + makeCMCall(payloadBuilder, newtoken, url); + // Reset payload builder and count for the next batch of records + payloadBuilder = new StringBuilder("{\"users\": ["); + count = 0; + } + } + + // If there are remaining records, print the payload + if (count > 0) { + payloadBuilder.deleteCharAt(payloadBuilder.length() - 1); // Remove the trailing comma + payloadBuilder.append("}"); + makeCMCall(payloadBuilder, newtoken, url); + if (debug) + System.out.println(payloadBuilder.toString()); + } + } catch (IOException e) { + e.printStackTrace(); + } + + return totalnbrofrecords; + + } + + /** + * Loads users from a file to the userset. Returns an int of number of users + * added. + *

+ * + * @param payloadBuilder payload for CM call that contains users to add + * @param newtoken jwt token to use + * @param url url to peform inserts + * @return int response code + */ + + public static int makeCMCall(StringBuilder payloadBuilder, String newtoken, String url) throws IOException { + + payloadBuilder.deleteCharAt(payloadBuilder.length() - 1); // Remove the trailing comma + payloadBuilder.append("]}"); + if (debug) + System.out.println(payloadBuilder.toString()); + String payload = payloadBuilder.toString(); + + // Create URL object + URL apiUrl = new URL(url); + // Create connection object + HttpURLConnection connection = (HttpURLConnection) apiUrl.openConnection(); + connection.setRequestMethod("POST"); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setDoOutput(true); + connection.setRequestProperty("Authorization", newtoken); + // Send request + OutputStream outputStream = connection.getOutputStream(); + outputStream.write(payload.getBytes()); + outputStream.flush(); + outputStream.close(); + + // Get response + int responseCode = connection.getResponseCode(); + BufferedReader reader; + if (responseCode >= 200 && responseCode < 300) { + reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + } else { + reader = new BufferedReader(new InputStreamReader(connection.getErrorStream())); + } + + // Read response + String lineresponse; + StringBuilder response = new StringBuilder(); + while ((lineresponse = reader.readLine()) != null) { + response.append(lineresponse); + } + reader.close(); + + // Print response + if (debug) { + System.out.println("Response Code: " + responseCode); + System.out.println("Response Body: " + response.toString()); + } + // Disconnect connection + connection.disconnect(); + // Reset payload builder and count for the next batch of records + payloadBuilder = new StringBuilder("{\"users\": ["); + + return responseCode; + + } + + /** + * Simple sample of showing how to load a couple of users to the userset. + * Returns an int of number of users added. + *

+ * + * @param url url to userset api + * @param newtoken jwt token to use + * + * @return int response code + */ + public static int addAUserToUserSet(String url, String newtoken) throws IOException { + boolean found = false; + + String payload = "{\"users\": [\"akhip@company.com\",\"user2@company.com\"]}"; + + // Create URL object + URL apiUrl = new URL(url); + // Create connection object + HttpURLConnection connection = (HttpURLConnection) apiUrl.openConnection(); + connection.setRequestMethod("POST"); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setDoOutput(true); + connection.setRequestProperty("Authorization", newtoken); + // Send request + OutputStream outputStream = connection.getOutputStream(); + outputStream.write(payload.getBytes()); + outputStream.flush(); + outputStream.close(); + + // Get response + int responseCode = connection.getResponseCode(); + BufferedReader reader; + if (responseCode >= 200 && responseCode < 300) { + reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + } else { + reader = new BufferedReader(new InputStreamReader(connection.getErrorStream())); + } + + // Read response + String line; + StringBuilder response = new StringBuilder(); + while ((line = reader.readLine()) != null) { + response.append(line); + } + reader.close(); + + // Print response + if (debug) { + System.out.println("Response Code: " + responseCode); + System.out.println("Response Body: " + response.toString()); + } + // Disconnect connection + connection.disconnect(); + + return responseCode; + + } + + public static String removeQuotes(String input) { + + // Remove double quotes from the input string + input = input.replace("\"", ""); + + return input; + } + + private static int numberOfLines(RandomAccessFile file) throws IOException { + int numberOfLines = 0; + while (file.readLine() != null) { + numberOfLines++; + } + file.seek(0); + return numberOfLines; + } + + /** + * Get JWT from CM + *

+ * + * @param apiUrl url to CM auth + * @param username username on CM + * @param pwd password on CM + * @return String jwt + */ + + public static String geAuthToken(String apiUrl, String usernb, String pwd) throws Exception + + { + + String jStr = "{\"username\":\"" + usernb + "\",\"password\":\"" + pwd + "\"}"; + disableCertValidation(); + + String totalstr = null; + try { + URL url = new URL(apiUrl); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestProperty("Content-length", String.valueOf(jStr.length())); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setRequestMethod("GET"); + connection.setDoOutput(true); + connection.setDoInput(true); + DataOutputStream output = new DataOutputStream(connection.getOutputStream()); + output.writeBytes(jStr); + output.close(); + BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + StringBuilder response = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + response.append(line); + } + reader.close(); + + JsonObject input = null; + JsonElement total = null; + JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject(); + if (rootNode.isJsonObject()) { + input = rootNode.getAsJsonObject(); + if (input.isJsonObject()) { + total = input.get("jwt"); + } + } + JsonPrimitive column = total.getAsJsonPrimitive(); + totalstr = column.getAsJsonPrimitive().toString(); + connection.disconnect(); + } catch (Exception e) { + e.printStackTrace(); + } + + return totalstr; + + } + + /** + * Get JWT from CM + *

+ * + * @param apiUrl url to CM auth + * @param username username on CM + * @param pwd password on CM + * @return String jwt + */ + + public static String geAuthToken(String apiUrl) throws Exception + + { + + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + String jStr = "{\"username\":\"" + userName + "\",\"password\":\"" + password + "\"}"; + + disableCertValidation(); + + String totalstr = null; + try { + URL url = new URL(apiUrl); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestProperty("Content-length", String.valueOf(jStr.length())); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setRequestMethod("GET"); + connection.setDoOutput(true); + connection.setDoInput(true); + DataOutputStream output = new DataOutputStream(connection.getOutputStream()); + output.writeBytes(jStr); + output.close(); + BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + StringBuilder response = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + response.append(line); + } + reader.close(); + if (debug) + System.out.println("response " + response); + JsonObject input = null; + JsonElement total = null; + JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject(); + if (rootNode.isJsonObject()) { + input = rootNode.getAsJsonObject(); + if (input.isJsonObject()) { + total = input.get("jwt"); + } + } + JsonPrimitive column = total.getAsJsonPrimitive(); + totalstr = column.getAsJsonPrimitive().toString(); + connection.disconnect(); + } catch (Exception e) { + e.printStackTrace(); + } + + return totalstr; + + } + + public static void disableCertValidation() throws Exception { + TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() { + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + + public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) { + } + + public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) { + } + } }; + + // Install the all-trusting trust manager + try { + SSLContext sc = SSLContext.getInstance("SSL"); + sc.init(null, trustAllCerts, new java.security.SecureRandom()); + HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); + } catch (Exception e) { + e.printStackTrace(); + } + + // Create all-trusting host name verifier + HostnameVerifier allHostsValid = new HostnameVerifier() { + public boolean verify(String hostname, SSLSession session) { + return true; + } + }; + + // Install the all-trusting host verifier + HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid); + } + +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/CustomException.java b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/CustomException.java new file mode 100644 index 00000000..6aed3bdd --- /dev/null +++ b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/CustomException.java @@ -0,0 +1,15 @@ +package com.example; + + +class CustomException extends Exception { + private int errorCode; + + public CustomException(String message, int errorCode) { + super(message); + this.errorCode = errorCode; + } + + public int getErrorCode() { + return errorCode; + } +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCADPBulkFPE.java b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCADPBulkFPE.java new file mode 100644 index 00000000..51a59b9a --- /dev/null +++ b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCADPBulkFPE.java @@ -0,0 +1,493 @@ +package com.example; + + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; + +import com.google.cloud.functions.HttpFunction; +import com.google.cloud.functions.HttpRequest; +import com.google.cloud.functions.HttpResponse; +import com.ingrian.security.nae.IngrianProvider; +import com.ingrian.security.nae.AbstractNAECipher; +import com.ingrian.security.nae.FPEParameterAndFormatSpec; +import com.ingrian.security.nae.NAEKey; +import com.ingrian.security.nae.NAESession; +import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder; +import com.ingrian.security.nae.IngrianProvider.Builder; +import com.ingrian.security.nae.NAECipher; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; + +import java.math.BigInteger; +import java.security.spec.AlgorithmParameterSpec; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Logger; +/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application + * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the + * data so applications or business intelligence tools do not have to change in order to use these columns. +* +* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested +* for all possible data sizes and combinations of encryption algorithms and IV, etc. +* Was tested with CM 2.14 & CADP 8.16.001 +* For more information on CADP see link below. +https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html +* For more information on Snowflake External Functions see link below. +https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp + +Notes: This example uses the CADP bulk API. +Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data +element has Unicode characters then the supported size limit would be 1750 characters + +Size of spec array should be same as the number of elements if user wants to use separate spec values +for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data +index. + * + *@author mwarner + * + */ +public class ThalesGCPSnowCADPBulkFPE implements HttpFunction { +// @Override + private static byte[][] data; + private static AlgorithmParameterSpec[] spec; + private static Integer[] snowrownbrs; + + private static int BATCHLIMIT = 10000; + private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPBulkFPE.class.getName()); + private static final Gson gson = new Gson(); + private static final String BADDATATAG = new String ("9999999999999999"); + + public void service(HttpRequest request, HttpResponse response) throws Exception { + + String tweakAlgo = null; + String tweakData = null; + String snowflakereturnstring = null; + JsonArray snowflakedata = null; + NAESession session = null; + + Map encryptedErrorMapTotal = new HashMap(); + JsonObject body = null; + int statusCode = 200; + int numberofchunks = 0; + + String keyName = "testfaas"; + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be + // returned when the user above does not have access to the key and if doing a lookup in the userset + // and the user does not exist. If returnciphertextforuserwithnokeyaccess = no then an error will be + // returned to the query, else the results set will provide ciphertext. + // yes/no + String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess"); + boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes"); + + // usersetlookup = should a userset lookup be done on the user from Cloud DB? + // yes/no + String usersetlookup = System.getenv("usersetlookup"); + // usersetidincm = should be the usersetid in CM to query. + String usersetID = System.getenv("usersetidincm"); + // usersetlookupip = this is the IP address to query the userset. Currently it + // is the userset in CM but could be a memcache or other in memory db. + String userSetLookupIP = System.getenv("usersetlookupip"); + boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes"); + String datatype = System.getenv("datatype"); + //mode of operation valid values are : encrypt or decrypt + String mode = System.getenv("mode"); + + int cipherType = 0; + if (mode.equals("encrypt")) + cipherType = Cipher.ENCRYPT_MODE; + else + cipherType = Cipher.DECRYPT_MODE; + + + int batchsize = Integer.parseInt(System.getenv("BATCHSIZE")); + + try { + + + + // This code is only to be used when input data contains user info. + /* + * if (usersetlookupbool) { // make sure cmuser is in Application Data + * Protection Clients Group + * + * boolean founduserinuserset = findUserInUserSet(bigquerysessionUser, userName, + * password, usersetID, userSetLookupIP); // System.out.println("Found User " + + * founduserinuserset); if (!founduserinuserset) throw new + * CustomException("1001, User Not in User Set", 1001); + * + * } else { usersetlookupbool = false; } + */ + //How many records in a chunk. Testing has indicated point of diminishing returns at 100 or 200, but + //may vary depending on size of data. + + try { + JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class); + JsonObject requestJson = null; + + if (requestParsed != null && requestParsed.isJsonObject()) { + requestJson = requestParsed.getAsJsonObject(); + } + + if (requestJson != null && requestJson.has("data")) { + snowflakedata = requestJson.getAsJsonArray("data"); + + } + + } catch (JsonParseException e) { + logger.severe("Error parsing JSON: " + e.getMessage()); + } + + // System.setProperty("com.ingrian.security.nae.IngrianNAE_Properties_Conf_Filename", + // "IngrianNAE.properties"); + System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", + "CADP_for_JAVA.properties"); + IngrianProvider builder = new Builder().addConfigFileInputStream( + getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build(); + // IngrianProvider builder = new + // Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build(); + session = NAESession.getSession(userName, password.toCharArray()); + NAEKey key = NAEKey.getSecretKey(keyName, session); + int row_number = 0; + + StringBuffer snowflakereturndata = new StringBuffer(); + int numberOfLines = snowflakedata.size(); + int totalRowsLeft = numberOfLines; + + if (batchsize > numberOfLines) + batchsize = numberOfLines; + if (batchsize >= BATCHLIMIT) + batchsize = BATCHLIMIT; + spec = new FPEParameterAndFormatSpec[batchsize]; + data = new byte[batchsize][]; + snowrownbrs = new Integer[batchsize]; + JsonObject bodyObject = new JsonObject(); + JsonArray dataArray = new JsonArray(); + JsonArray innerDataArray = new JsonArray(); + + String algorithm = null; + + if (datatype.equals("char")) + algorithm = "FPE/FF1/CARD62"; + else if (datatype.equals("charint")) + algorithm = "FPE/FF1/CARD10"; + else + algorithm = "FPE/FF1/CARD10"; + + + + AbstractNAECipher thalesCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider"); + int i = 0; + + if (batchsize > numberOfLines) + batchsize = numberOfLines; + if (batchsize >= BATCHLIMIT) + batchsize = BATCHLIMIT; + + FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo).build(); + + spec = new FPEParameterAndFormatSpec[batchsize]; + data = new byte[batchsize][]; + snowrownbrs = new Integer[batchsize]; + String sensitive = null; + thalesCipher.init(cipherType, key, spec[0]); + + //int i = 0; + int count = 0; + boolean newchunk = true; + int dataIndex = 0; + int specIndex = 0; + int snowRowIndex = 0; + //String sensitive = null; + + + while (i < numberOfLines) { + int index = 0; + + if (newchunk) { + + if (totalRowsLeft < batchsize) { + spec = new FPEParameterAndFormatSpec[totalRowsLeft]; + data = new byte[totalRowsLeft][]; + snowrownbrs = new Integer[totalRowsLeft]; + } else { + spec = new FPEParameterAndFormatSpec[batchsize]; + data = new byte[batchsize][]; + snowrownbrs = new Integer[batchsize]; + } + newchunk = false; + } + + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + for (int j = 0; j < snowflakerow.size(); j++) { + + if (j == 1) { + + sensitive = checkValid(snowflakerow); + + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + sensitive = BADDATATAG+sensitive; + data[dataIndex++] = sensitive.getBytes(); + // Can not return number since a leading 0 will not work. + // innerDataArray.add(new BigInteger(sensitive)); + } else + data[dataIndex++] = BADDATATAG.getBytes(); + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + data[dataIndex++] = sensitive.getBytes(); + } else { + data[dataIndex++] = sensitive.getBytes(); + } + spec[specIndex++] = param; + + } else { + data[dataIndex++] = sensitive.getBytes(); + spec[specIndex++] = param; + } + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + row_number = snowflakecolumn.getAsInt(); + snowrownbrs[snowRowIndex++] = row_number; + + } + + } + + if (count == batchsize - 1) { + + // Map to store exceptions while encryption + Map encryptedErrorMap = new HashMap(); + + byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap); + + for (Map.Entry entry : encryptedErrorMap.entrySet()) { + Integer mkey = entry.getKey(); + String mvalue = entry.getValue(); + encryptedErrorMapTotal.put(mkey, mvalue); + } + + for (int enc = 0; enc < encryptedData.length; enc++) { + + innerDataArray.add(snowrownbrs[enc]); + innerDataArray.add(new String(encryptedData[enc])); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + index++; + + } + + numberofchunks++; + newchunk = true; + count = 0; + dataIndex = 0; + specIndex = 0; + snowRowIndex = 0; + } else + count++; + + totalRowsLeft--; + i++; + } + if (count > 0) { + numberofchunks++; + int index = 0; + Map encryptedErrorMap = new HashMap(); + byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap); + for (int enc = 0; enc < encryptedData.length; enc++) { + innerDataArray.add(snowrownbrs[enc]); + innerDataArray.add(new String(encryptedData[enc])); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + index++; + + } + } + + + bodyObject.add("data", dataArray); + String bodyString = bodyObject.toString(); + + snowflakereturnstring = bodyString.toString(); + + + } catch ( + + Exception e) { + System.out.println("in exception with " + e.getMessage()); + if (returnciphertextbool) { + if (e.getMessage().contains("1401") || (e.getMessage().contains("1001") || (e.getMessage().contains("1002"))) ) { + + try { + snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, true, null,datatype); + } catch (IllegalBlockSizeException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } catch (BadPaddingException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } finally { + if (session != null) { + session.closeSession(); + } + } + System.out.println("number of chunks = " + numberofchunks); + //System.out.println("snowflakereturnstring = " + snowflakereturnstring); + response.getWriter().write(snowflakereturnstring); + + + + } + + public String formatReturnValue(int statusCode) + + { + StringBuffer snowflakereturndatasb = new StringBuffer(); + + snowflakereturndatasb.append("{ \"statusCode\":"); + snowflakereturndatasb.append(statusCode); + snowflakereturndatasb.append(","); + snowflakereturndatasb.append(" \"body\": {"); + snowflakereturndatasb.append(" \"data\": ["); + snowflakereturndatasb.append("] }}"); + System.out.println("in exception with "); + return snowflakereturndatasb.toString(); + } + + public String checkValid(JsonArray snowrow) { + String inputdata = null; + String notvalid = "notvalid"; + if (snowrow != null && snowrow.size() > 0) { + JsonElement element = snowrow.get(1); + if (element != null && !element.isJsonNull()) { + inputdata = element.getAsString(); + if (inputdata.isEmpty() || inputdata.length() < 2) { + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("Sensitive data is null or empty."); + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("bigquerytrow is null or empty."); + inputdata = notvalid + inputdata; + } + + return inputdata; + + } + + + + public String formatReturnValue(int statusCode, JsonArray snowflakedata, boolean error, Cipher thalesCipher, + String datatype) throws IllegalBlockSizeException, BadPaddingException { + int row_number = 0; + + String encdata = null; + String sensitive = null; + + JsonObject bodyObject = new JsonObject(); + JsonArray dataArray = new JsonArray(); + JsonArray innerDataArray = new JsonArray(); + + for (int i = 0; i < snowflakedata.size(); i++) { + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + for (int j = 0; j < snowflakerow.size(); j++) { + if (j == 1) { + sensitive = checkValid(snowflakerow); + + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + innerDataArray.add(sensitive); + // Can not return number since a leading 0 will not work. + // innerDataArray.add(new BigInteger(sensitive)); + } else + innerDataArray.add(BADDATATAG); + + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + innerDataArray.add(""); + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + innerDataArray.add(sensitive); + } else { + innerDataArray.add(sensitive); + } + + } else { + if (!error) { + byte[] outbuf = thalesCipher.doFinal(sensitive.getBytes()); + encdata = new String(outbuf); + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + innerDataArray.add(encdata); + // innerDataArray.add(new BigInteger(encdata)); + } else { + innerDataArray.add(encdata); + } + } else { + encdata = sensitive; + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + innerDataArray.add(encdata); + // innerDataArray.add(new BigInteger(encdata)); + } else { + innerDataArray.add(encdata); + } + } + } + + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + row_number = snowflakecolumn.getAsInt(); + innerDataArray.add(row_number); + } + } + } + + bodyObject.add("data", dataArray); + String bodyString = bodyObject.toString(); + + return bodyString; + + } + + public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID, + String userSetLookupIP) throws Exception { + + CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP); + + String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd); + String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken); + + boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken); + + return founduserinuserset; + + } +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCADPFPE.java b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCADPFPE.java new file mode 100644 index 00000000..aaa18fb2 --- /dev/null +++ b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCADPFPE.java @@ -0,0 +1,308 @@ +package com.example; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; + +import com.google.cloud.functions.HttpFunction; +import com.google.cloud.functions.HttpRequest; +import com.google.cloud.functions.HttpResponse; +import com.ingrian.security.nae.IngrianProvider; +import com.ingrian.security.nae.FPEParameterAndFormatSpec; +import com.ingrian.security.nae.NAEKey; +import com.ingrian.security.nae.NAESession; +import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder; +import com.ingrian.security.nae.IngrianProvider.Builder; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; + +import java.math.BigInteger; +import java.util.logging.Logger; + +/* This test app to test the logic for a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application + * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the + * data so applications or business intelligence tools do not have to change in order to use these columns. +* +* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested +* for all possible data sizes and combinations of encryption algorithms and IV, etc. +* Was tested with CM 2.14 & CADP 8.16 +* For more information on CADP see link below. +https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html +* For more information on Snowflake External Functions see link below. +https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp + * + *@author mwarner + * + */ +public class ThalesGCPSnowCADPFPE implements HttpFunction { +// @Override + + private static final Gson gson = new Gson(); + private static final String BADDATATAG = new String ("9999999999999999"); + + + public void service(HttpRequest request, HttpResponse response) throws Exception { + + String encdata = ""; + String tweakAlgo = null; + String tweakData = null; + String snowflakereturnstring = null; + JsonArray snowflakedata = null; + int statusCode = 200; + + NAESession session = null; + String keyName = "testfaas"; + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + + // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned + // when the user above does not have access to the key and if doing a + // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no + // then an error will be returned to the query, else the results set will provide ciphertext. + String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess"); + // yes,no + boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes"); + // usersetlookup = should a userset lookup be done on the user from Cloud DB + // yes,no + String usersetlookup = System.getenv("usersetlookup"); + // usersetidincm = should be the usersetid in CM to query. + String usersetID = System.getenv("usersetidincm"); + // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be + // a memcache or other in memory db. + String userSetLookupIP = System.getenv("usersetlookupip"); + boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes"); + String datatype = System.getenv("datatype"); + //mode of operation valid values are : encrypt or decrypt + String mode = System.getenv("mode"); + + int cipherType = 0; + if (mode.equals("encrypt")) + cipherType = Cipher.ENCRYPT_MODE; + else + cipherType = Cipher.DECRYPT_MODE; + + try { + + + //This code is only to be used when input data contains user info. + /* + * if (usersetlookupbool) { // make sure cmuser is in Application Data + * Protection Clients Group + * + * boolean founduserinuserset = findUserInUserSet(bigquerysessionUser, userName, + * password, usersetID, userSetLookupIP); // System.out.println("Found User " + + * founduserinuserset); if (!founduserinuserset) throw new + * CustomException("1001, User Not in User Set", 1001); + * + * } else { usersetlookupbool = false; } + */ + + + + try { + JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class); + JsonObject requestJson = null; + + if (requestParsed != null && requestParsed.isJsonObject()) { + requestJson = requestParsed.getAsJsonObject(); + } + + if (requestJson != null && requestJson.has("data")) { + snowflakedata = requestJson.getAsJsonArray("data"); + + } + + } catch (JsonParseException e) { + System.out.println("Error parsing JSON: " + e.getMessage()); + } + + + System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", + "CADP_for_JAVA.properties"); + IngrianProvider builder = new Builder().addConfigFileInputStream( + getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build(); + + session = NAESession.getSession(userName, password.toCharArray()); + NAEKey key = NAEKey.getSecretKey(keyName, session); + int row_number = 0; + + String algorithm = null; + + if (datatype.equals("char")) + algorithm = "FPE/FF1/CARD62"; + else if (datatype.equals("charint")) + algorithm = "FPE/FF1/CARD10"; + else + algorithm = "FPE/FF1/CARD10"; + + FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo) + .build(); + + Cipher thalesCipher = Cipher.getInstance(algorithm, "IngrianProvider"); + // initialize cipher to encrypt. + thalesCipher.init(cipherType, key, param); + + snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, false, thalesCipher, datatype); + + } catch (Exception e) { + + System.out.println("in exception with " + e.getMessage()); + if (returnciphertextbool) { + if (e.getMessage().contains("1401") + || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) { + + snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, true, null, datatype); + + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + + } finally { + if (session != null) { + session.closeSession(); + } + } + // System.out.println(snowflakereturnstring); + response.getWriter().write(snowflakereturnstring); + + } + + public String checkValid(JsonArray snowrow) { + String inputdata = null; + String notvalid = "notvalid"; + if (snowrow != null && snowrow.size() > 0) { + JsonElement element = snowrow.get(1); + if (element != null && !element.isJsonNull()) { + inputdata = element.getAsString(); + if (inputdata.isEmpty() || inputdata.length() < 2) { + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("Sensitive data is null or empty."); + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("bigquerytrow is null or empty."); + inputdata = notvalid + inputdata; + } + + return inputdata; + + } + + public String formatReturnValue(int statusCode) + + { + StringBuffer snowflakereturndatasb = new StringBuffer(); + + snowflakereturndatasb.append("{ \"statusCode\":"); + snowflakereturndatasb.append(statusCode); + snowflakereturndatasb.append(","); + snowflakereturndatasb.append(" \"body\": {"); + snowflakereturndatasb.append(" \"data\": ["); + snowflakereturndatasb.append("] }}"); + System.out.println("in exception with "); + return snowflakereturndatasb.toString(); + } + + public String formatReturnValue(int statusCode, JsonArray snowflakedata, boolean error, Cipher thalesCipher, + String datatype) throws IllegalBlockSizeException, BadPaddingException { + int row_number = 0; + + String encdata = null; + String sensitive = null; + + JsonObject bodyObject = new JsonObject(); + JsonArray dataArray = new JsonArray(); + JsonArray innerDataArray = new JsonArray(); + + for (int i = 0; i < snowflakedata.size(); i++) { + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + for (int j = 0; j < snowflakerow.size(); j++) { + if (j == 1) { + sensitive = checkValid(snowflakerow); + + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + innerDataArray.add(sensitive); + // Can not return number since a leading 0 will not work. + // innerDataArray.add(new BigInteger(sensitive)); + } else + innerDataArray.add(BADDATATAG); + + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + innerDataArray.add(""); + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + innerDataArray.add(sensitive); + } else { + innerDataArray.add(sensitive); + } + + } else { + if (!error) { + byte[] outbuf = thalesCipher.doFinal(sensitive.getBytes()); + encdata = new String(outbuf); + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + innerDataArray.add(encdata); + // innerDataArray.add(new BigInteger(encdata)); + } else { + innerDataArray.add(encdata); + } + } else { + encdata = sensitive; + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + innerDataArray.add(encdata); + // innerDataArray.add(new BigInteger(encdata)); + } else { + innerDataArray.add(encdata); + } + } + } + + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + row_number = snowflakecolumn.getAsInt(); + innerDataArray.add(row_number); + } + } + } + + bodyObject.add("data", dataArray); + String bodyString = bodyObject.toString(); + + return bodyString; + + } + + + public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID, + String userSetLookupIP) throws Exception { + + CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP); + + String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd); + String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken); + + boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken); + + return founduserinuserset; + + } +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCRDPBulkFPE.java b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCRDPBulkFPE.java new file mode 100644 index 00000000..a33ec2d7 --- /dev/null +++ b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCRDPBulkFPE.java @@ -0,0 +1,523 @@ +package com.example; + +import com.google.cloud.functions.HttpFunction; +import com.google.cloud.functions.HttpRequest; +import com.google.cloud.functions.HttpResponse; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; + +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +import java.util.HashMap; +import java.util.Map; + + +/* This is a Thales CRDP UDF for Snowflake. It uses the Thales CRDP Bulk API. + * It is an example of how to use Thales CipherTrust REST Application Dataprotection (CRDP) + * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the + * data so applications or business intelligence tools do not have to change in order to use these columns. +* +* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested +* for all possible data sizes and combinations of encryption algorithms and IV, etc. +* Was tested with CM 2.14 and CRDP 1.0 +* For more information on CRDP see link below. +https://thalesdocs.com/ctp/con/crdp/latest/index.html +* For more information on Snowflake External Functions see link below. +https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp + * + *@author mwarner + * + */ +public class ThalesGCPSnowCRDPBulkFPE implements HttpFunction { +// @Override + + private static final Gson gson = new Gson(); + private static final String BADDATATAG = new String ("9999999999999999"); + private static int BATCHLIMIT = 10000; + private static final String REVEALRETURNTAG = new String("data"); + private static final String PROTECTRETURNTAG = new String("protected_data"); + + + public void service(HttpRequest request, HttpResponse response) throws Exception { + Map snowErrorMap = new HashMap(); + String encdata = ""; + int error_count = 0; + int statusCode = 200; + + + JsonObject result = new JsonObject(); + JsonArray replies = new JsonArray(); + JsonArray rownbranddata = new JsonArray(); + + String keyName = "testfaas"; + String crdpip = System.getenv("CRDPIP"); + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + + // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned + // when the user above does not have access to the key and if doing a + // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no + // then an error will be returned to the query, else the results set will provide ciphertext. + String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess"); + // yes,no + boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes"); + // usersetlookup = should a userset lookup be done on the user from Cloud DB + // yes,no + String usersetlookup = System.getenv("usersetlookup"); + // usersetidincm = should be the usersetid in CM to query. + String usersetID = System.getenv("usersetidincm"); + // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be + // a memcache or other in memory db. + String userSetLookupIP = System.getenv("usersetlookupip"); + boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes"); + String keymetadatalocation = System.getenv("keymetadatalocation"); + String external_version_from_ext_source = System.getenv("keymetadata"); + String protection_profile = System.getenv("protection_profile"); + String mode = System.getenv("mode"); + String datatype = System.getenv("datatype"); + + String inputDataKey = null; + String outputDataKey = null; + String protectedData = null; + String externalkeymetadata = null; + String jsonBody = null; + String jsonTagForProtectReveal = null; + + boolean bad_data = false; + if (mode.equals("protectbulk")) { + inputDataKey = "data_array"; + outputDataKey = "protected_data_array"; + jsonTagForProtectReveal = PROTECTRETURNTAG; + } else { + inputDataKey = "protected_data_array"; + outputDataKey = "data_array"; + jsonTagForProtectReveal = REVEALRETURNTAG; + } + + String snowflakeuser = "snowflakeuser"; + JsonArray snowflakedata = null; + String snowflakereturnstring = null; + Response crdp_response = null; + try { + + + // This code is only to be used when input data contains user info. + /* + * if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group + * + * boolean founduserinuserset = findUserInUserSet(bigquerysessionUser, userName, password, usersetID, + * userSetLookupIP); // System.out.println("Found User " + founduserinuserset); if (!founduserinuserset) + * throw new CustomException("1001, User Not in User Set", 1001); + * + * } else { usersetlookupbool = false; } + */ + + + + try { + JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class); + JsonObject requestJson = null; + + if (requestParsed != null && requestParsed.isJsonObject()) { + requestJson = requestParsed.getAsJsonObject(); + } + + if (requestJson != null && requestJson.has("data")) { + snowflakedata = requestJson.getAsJsonArray("data"); + + } + + } catch (JsonParseException e) { + System.out.println("Error parsing JSON: " + e.getMessage()); + //logger.severe("Error parsing JSON: " + e.getMessage()); + } + + // Serialization + + int numberofchunks = 0; + StringBuffer protection_policy_buff = new StringBuffer(); + String notvalid = "notvalid"; + // StringBuffer snowflakereturndata = new StringBuffer(); + + int numberOfLines = snowflakedata.size(); + int totalRowsLeft = numberOfLines; + int batchsize = Integer.parseInt(System.getenv("BATCHSIZE")); + + if (batchsize > numberOfLines) + batchsize = numberOfLines; + if (batchsize >= BATCHLIMIT) + batchsize = BATCHLIMIT; + // Serialization + + String crdpjsonBody = null; + // String external_version_from_ext_source = "1004001"; + // String external_version_from_ext_source = "1001001"; + + int i = 0; + int count = 0; + int totalcount = 0; + + int dataIndex = 0; // assumes index from snowflake will always be sequential. + JsonObject crdp_payload = new JsonObject(); + String sensitive = null; + JsonArray crdp_payload_array = new JsonArray(); + + OkHttpClient client = new OkHttpClient().newBuilder().build(); + MediaType mediaType = MediaType.parse("application/json"); + String urlStr = "http://" + crdpip + ":8090/v1/" + mode; + + while (i < numberOfLines) { + + for (int b = 0; b < batchsize && b < totalRowsLeft; b++) { + + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + sensitive = checkValid(snowflakerow); + protection_profile = protection_profile.trim(); + // Format the output + String formattedElement = String.format("\"protection_policy_name\" : \"%s\"", protection_profile); + protection_policy_buff.append(formattedElement); + protection_policy_buff.append(","); + + if (mode.equals("protectbulk")) { + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + // System.out.println("adding null not charint or nbr"); + sensitive = sensitive.replace("notvalid", ""); + sensitive = BADDATATAG + sensitive; + } else + sensitive = BADDATATAG; + + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + + } else if (sensitive.contains("notvalid")) { + // sensitive = sensitive.replace("notvalid", ""); + + } + encdata = sensitive; + + } + crdp_payload_array.add(sensitive); + } else { + JsonObject protectedDataObject = new JsonObject(); + protectedDataObject.addProperty(PROTECTRETURNTAG, sensitive); + if (keymetadatalocation.equalsIgnoreCase("external")) { + protectedDataObject.addProperty("external_version", external_version_from_ext_source); + } + crdp_payload_array.add(protectedDataObject); + + } + + + if (count == batchsize - 1) { + crdp_payload.add(inputDataKey, crdp_payload_array); + String inputdataarray = null; + if (mode.equals("revealbulk")) { + crdp_payload.addProperty("username", snowflakeuser); + inputdataarray = crdp_payload.toString(); + protection_policy_buff.append(inputdataarray); + jsonBody = protection_policy_buff.toString(); + jsonBody = jsonBody.replaceFirst("\\{", " "); + + } else { + inputdataarray = crdp_payload.toString(); + protection_policy_buff.append(inputdataarray); + inputdataarray = protection_policy_buff.toString(); + jsonBody = inputdataarray.replace("{", " "); + } + jsonBody = "{" + jsonBody; + + RequestBody body = RequestBody.create(mediaType, jsonBody); + + Request crdp_request = new Request.Builder().url(urlStr).method("POST", body) + .addHeader("Content-Type", "application/json").build(); + crdp_response = client.newCall(crdp_request).execute(); + String crdpreturnstr = null; + if (crdp_response.isSuccessful()) { + // Parse JSON response + String responseBody = crdp_response.body().string(); + JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class); + JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey); + + String status = jsonObject.get("status").getAsString(); + int success_count = jsonObject.get("success_count").getAsInt(); + error_count = jsonObject.get("error_count").getAsInt(); + if (error_count > 0) + System.out.println("errors " + error_count); + + for (JsonElement element : protectedDataArray) { + + JsonObject protectedDataObject = element.getAsJsonObject(); + if (protectedDataObject.has(jsonTagForProtectReveal)) { + + protectedData = protectedDataObject.get(jsonTagForProtectReveal).getAsString(); + //System.out.println(protectedData); + + rownbranddata.add(dataIndex); + rownbranddata.add(new String(protectedData)); + replies.add(rownbranddata); + rownbranddata = new JsonArray(); + if (mode.equals("protectbulk")) { + if (keymetadatalocation.equalsIgnoreCase("external") && mode.equalsIgnoreCase("protect")) { + externalkeymetadata = protectedDataObject.get("external_version") + .getAsString(); + // System.out.println("Protected Data ext key metadata need to store this: " + // + externalkeymetadata); + + } + } + } else if (protectedDataObject.has("error_message")) { + String errorMessage = protectedDataObject.get("error_message").getAsString(); + System.out.println("error_message: " + errorMessage); + snowErrorMap.put(i, errorMessage); + bad_data = true; + } else + System.out.println("unexpected json value from results: "); + dataIndex++; + + } + + crdp_payload_array = new JsonArray(); + protection_policy_buff = new StringBuffer(); + numberofchunks++; + totalcount = totalcount + count; + count = 0; + } else {// throw error.... + System.err.println("Request failed with status code: " + crdp_response.code()); + throw new CustomException("1010, Unexpected Error ", 1010); + } + + } else { + count++; + } + totalRowsLeft--; + i++; + } + } + if (count > 0) { + crdp_payload.add(inputDataKey, crdp_payload_array); + String inputdataarray = null; + if (mode.equals("revealbulk")) { + crdp_payload.addProperty("username", snowflakeuser); + inputdataarray = crdp_payload.toString(); + protection_policy_buff.append(inputdataarray); + jsonBody = protection_policy_buff.toString(); + jsonBody = jsonBody.replaceFirst("\\{", " "); + + } else { + inputdataarray = crdp_payload.toString(); + protection_policy_buff.append(inputdataarray); + inputdataarray = protection_policy_buff.toString(); + jsonBody = inputdataarray.replace("{", " "); + } + jsonBody = "{" + jsonBody; + + RequestBody body = RequestBody.create(mediaType, jsonBody); + + Request crdp_request = new Request.Builder().url(urlStr).method("POST", body) + .addHeader("Content-Type", "application/json").build(); + crdp_response = client.newCall(crdp_request).execute(); + String crdpreturnstr = null; + if (crdp_response.isSuccessful()) { + // Parse JSON response + String responseBody = crdp_response.body().string(); + JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class); + JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey); + + String status = jsonObject.get("status").getAsString(); + int success_count = jsonObject.get("success_count").getAsInt(); + error_count = jsonObject.get("error_count").getAsInt(); + if (error_count > 0) + System.out.println("errors " + error_count); + + for (JsonElement element : protectedDataArray) { + + JsonObject protectedDataObject = element.getAsJsonObject(); + if (protectedDataObject.has(jsonTagForProtectReveal)) { + + protectedData = protectedDataObject.get(jsonTagForProtectReveal).getAsString(); + // System.out.println(protectedData); + + rownbranddata.add(dataIndex); + rownbranddata.add(new String(protectedData)); + replies.add(rownbranddata); + rownbranddata = new JsonArray(); + if (mode.equals("protectbulk")) { + if (keymetadatalocation.equalsIgnoreCase("external") && mode.equalsIgnoreCase("protect")) { + externalkeymetadata = protectedDataObject.get("external_version").getAsString(); + // System.out.println("Protected Data ext key metadata need to store this: " + // + externalkeymetadata); + + } + } + } else if (protectedDataObject.has("error_message")) { + String errorMessage = protectedDataObject.get("error_message").getAsString(); + System.out.println("error_message: " + errorMessage); + snowErrorMap.put(i, errorMessage); + bad_data = true; + } else + System.out.println("unexpected json value from results: "); + dataIndex++; + + } + + crdp_payload_array = new JsonArray(); + protection_policy_buff = new StringBuffer(); + numberofchunks++; + totalcount = totalcount + count; + count = 0; + + } else { + System.err.println("Request failed with status code: " + crdp_response.code()); + } + } + crdp_response.close(); + System.out.println("total chuncks " + numberofchunks); + + result.add("data", replies); + + snowflakereturnstring = result.toString(); + + } catch (Exception e) { + System.out.println("in exception with " + e.getMessage()); + snowflakereturnstring = "exception "; + if (returnciphertextbool) { + if (e.getMessage().contains("1401") + || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) { + + result = new JsonObject(); + replies = new JsonArray(); + rownbranddata = new JsonArray(); + + for (int i = 0; i < snowflakedata.size(); i++) { + + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + for (int j = 0; j < snowflakerow.size(); j++) { + if (j == 1) { + // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); + // FPE example + String sensitive = checkValid(snowflakerow); + + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + } else + sensitive = BADDATATAG; + + } else if (sensitive.equalsIgnoreCase("null") + || sensitive.equalsIgnoreCase("notvalid")) { + + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + + } + encdata = sensitive; + + } else { + //System.out.println("normal number data" + sensitive); + } + rownbranddata.add(sensitive); + replies.add(rownbranddata); + rownbranddata = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + int row_number = snowflakecolumn.getAsInt(); + rownbranddata.add(row_number); + } + } + } + + result.add("data", replies); + JsonObject inputJsonObject = new JsonObject(); + String bodyString = result.toString(); + inputJsonObject.addProperty("statusCode", 200); + inputJsonObject.addProperty("body", bodyString); + + snowflakereturnstring = inputJsonObject.toString(); + + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + + } finally { + + } + + //System.out.println("return string " + snowflakereturnstring); + response.getWriter().write(snowflakereturnstring); + + } + + public String formatReturnValue(int statusCode) + + { + StringBuffer snowflakereturndatasb = new StringBuffer(); + + snowflakereturndatasb.append("{ \"statusCode\":"); + snowflakereturndatasb.append(statusCode); + snowflakereturndatasb.append(","); + snowflakereturndatasb.append(" \"body\": {"); + snowflakereturndatasb.append(" \"data\": ["); + snowflakereturndatasb.append("] }}"); + System.out.println("in exception with "); + return snowflakereturndatasb.toString(); + } + + + public String checkValid(JsonArray snowrow) { + String inputdata = null; + String notvalid = "notvalid"; + if (snowrow != null && snowrow.size() > 0) { + JsonElement element = snowrow.get(1); + if (element != null && !element.isJsonNull()) { + inputdata = element.getAsString(); + if (inputdata.isEmpty() || inputdata.length() < 2) { + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("Sensitive data is null or empty."); + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("bigquerytrow is null or empty."); + inputdata = notvalid + inputdata; + } + + return inputdata; + + } + + public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID, + String userSetLookupIP) throws Exception { + + CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP); + + String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd); + String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken); + + boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken); + + return founduserinuserset; + + } + +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCRDPFPE.java b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCRDPFPE.java new file mode 100644 index 00000000..7b96e329 --- /dev/null +++ b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCRDPFPE.java @@ -0,0 +1,371 @@ +package com.example; + +import com.google.cloud.functions.HttpFunction; +import com.google.cloud.functions.HttpRequest; +import com.google.cloud.functions.HttpResponse; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; + +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +import java.util.HashMap; +import java.util.Map; + +/* This is a Thales CRDP UDF for Snowflake. + * It is an example of how to use Thales CipherTrust REST Application Dataprotection (CRDP) + * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the + * data so applications or business intelligence tools do not have to change in order to use these columns. +* +* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested +* for all possible data sizes and combinations of encryption algorithms and IV, etc. +* Was tested with CM 2.14 and CRDP 1.0 +* For more information on CRDP see link below. +https://thalesdocs.com/ctp/con/crdp/latest/index.html +* For more information on Snowflake External Functions see link below. +https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp + * + *@author mwarner + * + */ +public class ThalesGCPSnowCRDPFPE implements HttpFunction { +// @Override + + private static final Gson gson = new Gson(); + private static final String BADDATATAG = new String("9999999999999999"); + private static final String REVEALRETURNTAG = new String("data"); + private static final String PROTECTRETURNTAG = new String("protected_data"); + + public void service(HttpRequest request, HttpResponse response) throws Exception { + Map snowErrorMap = new HashMap(); + String encdata = ""; + String snowflakereturnstring = null; + int statusCode = 200; + JsonObject result = new JsonObject(); + JsonArray replies = new JsonArray(); + JsonArray rownbranddata = new JsonArray(); + + String keyName = "testfaas"; + String crdpip = System.getenv("CRDPIP"); + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + + // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned + // when the user above does not have access to the key and if doing a + // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no + // then an error will be returned to the query, else the results set will provide ciphertext. + String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess"); + // yes,no + boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes"); + // usersetlookup = should a userset lookup be done on the user from Cloud DB + // yes,no + String usersetlookup = System.getenv("usersetlookup"); + // usersetidincm = should be the usersetid in CM to query. + String usersetID = System.getenv("usersetidincm"); + // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be + // a memcache or other in memory db. + String userSetLookupIP = System.getenv("usersetlookupip"); + boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes"); + String keymetadatalocation = System.getenv("keymetadatalocation"); + String external_version_from_ext_source = System.getenv("keymetadata"); + String protection_profile = System.getenv("protection_profile"); + String mode = System.getenv("mode"); + String datatype = System.getenv("datatype"); + String dataKey = null; + + boolean bad_data = false; + String jsonTagForProtectReveal = null; + + if (mode.equals("protect")) { + dataKey = "data"; + jsonTagForProtectReveal = PROTECTRETURNTAG; + } else { + dataKey = "protected_data"; + jsonTagForProtectReveal = REVEALRETURNTAG; + } + + String snowflakeuser = "snowflakeuser"; + JsonArray snowflakedata = null; + + try { + + // This code is only to be used when input data contains user info. + /* + * if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group + * + * boolean founduserinuserset = findUserInUserSet(bigquerysessionUser, userName, password, usersetID, + * userSetLookupIP); // System.out.println("Found User " + founduserinuserset); if (!founduserinuserset) + * throw new CustomException("1001, User Not in User Set", 1001); + * + * } else { usersetlookupbool = false; } + */ + + try { + JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class); + JsonObject requestJson = null; + + if (requestParsed != null && requestParsed.isJsonObject()) { + requestJson = requestParsed.getAsJsonObject(); + } + + if (requestJson != null && requestJson.has("data")) { + snowflakedata = requestJson.getAsJsonArray("data"); + + } + + } catch (JsonParseException e) { + System.out.println("Error parsing JSON: " + e.getMessage()); + + } + + int row_number = 0; + + String protectedData = null; + String externalkeymetadata = null; + String crdpjsonBody = null; + // String external_version_from_ext_source = "1004001"; + // String external_version_from_ext_source = "1001001"; + + JsonObject crdp_payload = new JsonObject(); + + crdp_payload.addProperty("protection_policy_name", protection_profile); + + String sensitive = null; + OkHttpClient client = new OkHttpClient().newBuilder().build(); + MediaType mediaType = MediaType.parse("application/json"); + String urlStr = "http://" + crdpip + ":8090/v1/" + mode; + + // encrypt data + for (int i = 0; i < snowflakedata.size(); i++) { + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + for (int j = 0; j < snowflakerow.size(); j++) { + + if (j == 1) { + // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); + // FPE example + sensitive = checkValid(snowflakerow); + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + } else + sensitive = BADDATATAG; + + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + + } + encdata = sensitive; + + } else { + + crdp_payload.addProperty(dataKey, sensitive); + if (mode.equals("reveal")) { + crdp_payload.addProperty("username", snowflakeuser); + if (keymetadatalocation.equalsIgnoreCase("external")) { + crdp_payload.addProperty("external_version", external_version_from_ext_source); + } + } + crdpjsonBody = crdp_payload.toString(); + // System.out.println(crdpjsonBody); + RequestBody body = RequestBody.create(mediaType, crdpjsonBody); + + Request crdp_request = new Request.Builder().url(urlStr).method("POST", body) + .addHeader("Content-Type", "application/json").build(); + Response crdp_response = client.newCall(crdp_request).execute(); + + if (crdp_response.isSuccessful()) { + + // Parse JSON response + String responseBody = crdp_response.body().string(); + Gson gson = new Gson(); + JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class); + + if (jsonObject.has(jsonTagForProtectReveal)) { + protectedData = jsonObject.get(jsonTagForProtectReveal).getAsString(); + if (keymetadatalocation.equalsIgnoreCase("external") && mode.equalsIgnoreCase("protect")) { + externalkeymetadata = jsonObject.get("external_version").getAsString(); + // System.out.println("Protected Data ext key metadata need to store this: " + // + externalkeymetadata); + } + } else if (jsonObject.has("error_message")) { + String errorMessage = jsonObject.get("error_message").getAsString(); + System.out.println("error_message: " + errorMessage); + snowErrorMap.put(i, errorMessage); + bad_data = true; + } else + System.out.println("unexpected json value from results: "); + } else { + System.err.println("Request failed with status code: " + crdp_response.code()); + } + + crdp_response.close(); + + encdata = protectedData; + + } + + rownbranddata.add(encdata); + replies.add(rownbranddata); + rownbranddata = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + row_number = snowflakecolumn.getAsInt(); + rownbranddata.add(row_number); + + } + + } + + } + + result.add("data", replies); + snowflakereturnstring = result.toString(); + + } catch (Exception e) { + + System.out.println("in exception with " + e.getMessage()); + snowflakereturnstring = "exception "; + if (returnciphertextbool) { + if (e.getMessage().contains("1401") + || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) { + + result = new JsonObject(); + replies = new JsonArray(); + rownbranddata = new JsonArray(); + + for (int i = 0; i < snowflakedata.size(); i++) { + + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + for (int j = 0; j < snowflakerow.size(); j++) { + if (j == 1) { + // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); + // FPE example + String sensitive = checkValid(snowflakerow); + + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + // System.out.println("adding null not charint or nbr"); + sensitive = sensitive.replace("notvalid", ""); + } else + sensitive = BADDATATAG; + + } else if (sensitive.equalsIgnoreCase("null") + || sensitive.equalsIgnoreCase("notvalid")) { + + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + + } + encdata = sensitive; + + } else { + // System.out.println("normal number data" + sensitive); + } + rownbranddata.add(sensitive); + replies.add(rownbranddata); + rownbranddata = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + int row_number = snowflakecolumn.getAsInt(); + rownbranddata.add(row_number); + } + } + } + + result.add("data", replies); + JsonObject inputJsonObject = new JsonObject(); + String bodyString = result.toString(); + inputJsonObject.addProperty("statusCode", 200); + inputJsonObject.addProperty("body", bodyString); + + snowflakereturnstring = inputJsonObject.toString(); + // System.out.println(" new data " + snowflakereturnstring); + + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + + } finally { + + } + // System.out.println(snowflakereturnstring); + response.getWriter().write(snowflakereturnstring); + + } + + public String formatReturnValue(int statusCode) + + { + StringBuffer snowflakereturndatasb = new StringBuffer(); + + snowflakereturndatasb.append("{ \"statusCode\":"); + snowflakereturndatasb.append(statusCode); + snowflakereturndatasb.append(","); + snowflakereturndatasb.append(" \"body\": {"); + snowflakereturndatasb.append(" \"data\": ["); + snowflakereturndatasb.append("] }}"); + System.out.println("in exception with "); + return snowflakereturndatasb.toString(); + } + + public String checkValid(JsonArray snowrow) { + String inputdata = null; + String notvalid = "notvalid"; + if (snowrow != null && snowrow.size() > 0) { + JsonElement element = snowrow.get(1); + if (element != null && !element.isJsonNull()) { + inputdata = element.getAsString(); + if (inputdata.isEmpty() || inputdata.length() < 2) { + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("Sensitive data is null or empty."); + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("bigquerytrow is null or empty."); + inputdata = notvalid + inputdata; + } + + return inputdata; + + } + + public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID, + String userSetLookupIP) throws Exception { + + CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP); + + String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd); + String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken); + + boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken); + + return founduserinuserset; + + } + +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/CMUserSetHelper.java b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/CMUserSetHelper.java new file mode 100644 index 00000000..92609666 --- /dev/null +++ b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/CMUserSetHelper.java @@ -0,0 +1,508 @@ + + +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; + +import java.io.BufferedReader; +import java.io.DataOutputStream; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.security.KeyStore; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; +/* + * This app provides a number of different helper methods dealing with CM Application Data Protection UserSets. There is a method to find + * a user in a userset and another method to populate the userset from a flat file. Usersets are typically used within + * an access policy but they are not restricted to that usage. + * Was tested with CM 2.14 + * Note: This source code is only to be used for testing and proof of concepts. + * @author mwarner + * + */ + +public class CMUserSetHelper { + + static String hostnamevalidate = "yourhostname"; + + String usersetid = "716f01a6-5cab-4799-925a-6dc2d8712fc1"; + String cmIP = "yourip"; + String apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?name="; + // + username_in_userset; + String addusertouserset = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users"; + + int totalrecords = 0; + String authUrl = "https://" + cmIP + "/api/v1/auth/tokens"; + + static boolean debug = true; + int chunksize = 5; + static int CHUNKSIZEMAX = 100; + + public CMUserSetHelper(String usersetid, String cmIP) { + + this.usersetid = usersetid; + this.cmIP = cmIP; + this.apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?name="; + // this.apiUrlGetUsers = "https://" + cmIP + + // "/api/v1/data-protection/user-sets/" + usersetid + "/users?limit=" + + // this.usersetlimit + "&name="; + this.addusertouserset = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users"; + this.authUrl = "https://" + cmIP + "/api/v1/auth/tokens"; + } + + public static void main(String[] args) throws Exception { + + String username = args[0]; + String password = args[1]; + String cmip = args[2]; + String usersetid = args[3]; + String filePath = args[4]; + + CMUserSetHelper cmusersetHelper = new CMUserSetHelper(usersetid, cmip); + int totalrecords = 0; + + // String apiUrl = apiUrlGetUsers; + String jwthtoken = geAuthToken(cmusersetHelper.authUrl, username, password); + + String newtoken = "Bearer " + removeQuotes(jwthtoken); + + // String filePath = + // "C:\\Users\\t0185905\\workspace\\CT-VL-GCP\\src\\main\\java\\com\\example\\emailAddresses.txt"; + RandomAccessFile file = new RandomAccessFile(filePath, "r"); + if (cmusersetHelper.chunksize > CHUNKSIZEMAX) + cmusersetHelper.chunksize = CHUNKSIZEMAX; + int totoalnbrofrecords = numberOfLines(file); + if (cmusersetHelper.chunksize > totoalnbrofrecords) { + cmusersetHelper.chunksize = totoalnbrofrecords / 2; + } + totalrecords = cmusersetHelper.addAUserToUserSetFromFile(cmusersetHelper.addusertouserset, newtoken, filePath); + System.out.println("Totalrecords inserted into Userset " + cmusersetHelper.usersetid + " = " + totalrecords); + + } + + /** + * Returns an boolean if user found + *

+ * + * @param user user to find + * @param newtoken jwt token to use + * @return boolean true if found in userset + * @throws CustomException + */ + public boolean findUserInUserSet(String user, String newtoken) throws CustomException { + boolean found = false; + + String apiUrl = this.apiUrlGetUsers + user; + + apiUrl = removeQuotes(apiUrl); + + try { + URL url = new URL(apiUrl); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("GET"); + connection.setRequestProperty("Authorization", newtoken); + connection.setRequestProperty("accept", "application/json"); + + connection.setDoInput(true); + BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + + StringBuilder response = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + response.append(line); + } + reader.close(); + + Gson gson = new Gson(); + if (debug) + System.out.println("response " + response); + JsonObject input = null; + JsonElement total = null; + JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject(); + if (rootNode.isJsonObject()) { + input = rootNode.getAsJsonObject(); + if (input.isJsonObject()) { + total = input.get("total"); + } + } + JsonPrimitive column = total.getAsJsonPrimitive(); + String totalstr = column.getAsJsonPrimitive().toString(); + + Integer i = Integer.valueOf(totalstr); + + if (i > 0) { + found = true; + + } + connection.disconnect(); + } catch (Exception e) { + if (e.getMessage().contains("403")) { + throw new CustomException("1002, User Not in Application Data Protection Clients ", 1002); + } else + e.printStackTrace(); + } + + return found; + + } + + /** + * Loads users from a file to the userset. Returns an int of number of users + * added. + *

+ * + * @param url url to userset api + * @param newtoken jwt token to use + * @param filePath file to load + * @return int totalnumberofrecords added to userset + */ + + public int addAUserToUserSetFromFile(String url, String newtoken, String filePath) throws IOException { + + int totalnbrofrecords = 0; + try (BufferedReader br = new BufferedReader(new FileReader(filePath))) { + String line; + int count = 0; + StringBuilder payloadBuilder = new StringBuilder(); + + payloadBuilder.append("{\"users\": ["); + + while ((line = br.readLine()) != null) { + totalnbrofrecords++; + // Append the email address to the payload + payloadBuilder.append("\"").append(line).append("\","); + count++; + + // If 'n' records have been read, print the payload + if (count == chunksize) { + makeCMCall(payloadBuilder, newtoken, url); + // Reset payload builder and count for the next batch of records + payloadBuilder = new StringBuilder("{\"users\": ["); + count = 0; + } + } + + // If there are remaining records, print the payload + if (count > 0) { + payloadBuilder.deleteCharAt(payloadBuilder.length() - 1); // Remove the trailing comma + payloadBuilder.append("}"); + makeCMCall(payloadBuilder, newtoken, url); + if (debug) + System.out.println(payloadBuilder.toString()); + } + } catch (IOException e) { + e.printStackTrace(); + } + + return totalnbrofrecords; + + } + + /** + * Loads users from a file to the userset. Returns an int of number of users + * added. + *

+ * + * @param payloadBuilder payload for CM call that contains users to add + * @param newtoken jwt token to use + * @param url url to peform inserts + * @return int response code + */ + + public static int makeCMCall(StringBuilder payloadBuilder, String newtoken, String url) throws IOException { + + payloadBuilder.deleteCharAt(payloadBuilder.length() - 1); // Remove the trailing comma + payloadBuilder.append("]}"); + if (debug) + System.out.println(payloadBuilder.toString()); + String payload = payloadBuilder.toString(); + + // Create URL object + URL apiUrl = new URL(url); + // Create connection object + HttpURLConnection connection = (HttpURLConnection) apiUrl.openConnection(); + connection.setRequestMethod("POST"); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setDoOutput(true); + connection.setRequestProperty("Authorization", newtoken); + // Send request + OutputStream outputStream = connection.getOutputStream(); + outputStream.write(payload.getBytes()); + outputStream.flush(); + outputStream.close(); + + // Get response + int responseCode = connection.getResponseCode(); + BufferedReader reader; + if (responseCode >= 200 && responseCode < 300) { + reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + } else { + reader = new BufferedReader(new InputStreamReader(connection.getErrorStream())); + } + + // Read response + String lineresponse; + StringBuilder response = new StringBuilder(); + while ((lineresponse = reader.readLine()) != null) { + response.append(lineresponse); + } + reader.close(); + + // Print response + if (debug) { + System.out.println("Response Code: " + responseCode); + System.out.println("Response Body: " + response.toString()); + } + // Disconnect connection + connection.disconnect(); + // Reset payload builder and count for the next batch of records + payloadBuilder = new StringBuilder("{\"users\": ["); + + return responseCode; + + } + + /** + * Simple sample of showing how to load a couple of users to the userset. + * Returns an int of number of users added. + *

+ * + * @param url url to userset api + * @param newtoken jwt token to use + * + * @return int response code + */ + public static int addAUserToUserSet(String url, String newtoken) throws IOException { + boolean found = false; + + String payload = "{\"users\": [\"akhip@company.com\",\"user2@company.com\"]}"; + + // Create URL object + URL apiUrl = new URL(url); + // Create connection object + HttpURLConnection connection = (HttpURLConnection) apiUrl.openConnection(); + connection.setRequestMethod("POST"); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setDoOutput(true); + connection.setRequestProperty("Authorization", newtoken); + // Send request + OutputStream outputStream = connection.getOutputStream(); + outputStream.write(payload.getBytes()); + outputStream.flush(); + outputStream.close(); + + // Get response + int responseCode = connection.getResponseCode(); + BufferedReader reader; + if (responseCode >= 200 && responseCode < 300) { + reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + } else { + reader = new BufferedReader(new InputStreamReader(connection.getErrorStream())); + } + + // Read response + String line; + StringBuilder response = new StringBuilder(); + while ((line = reader.readLine()) != null) { + response.append(line); + } + reader.close(); + + // Print response + if (debug) { + System.out.println("Response Code: " + responseCode); + System.out.println("Response Body: " + response.toString()); + } + // Disconnect connection + connection.disconnect(); + + return responseCode; + + } + + public static String removeQuotes(String input) { + + // Remove double quotes from the input string + input = input.replace("\"", ""); + + return input; + } + + private static int numberOfLines(RandomAccessFile file) throws IOException { + int numberOfLines = 0; + while (file.readLine() != null) { + numberOfLines++; + } + file.seek(0); + return numberOfLines; + } + + /** + * Get JWT from CM + *

+ * + * @param apiUrl url to CM auth + * @param username username on CM + * @param pwd password on CM + * @return String jwt + */ + + public static String geAuthToken(String apiUrl, String usernb, String pwd) throws Exception + + { + + String jStr = "{\"username\":\"" + usernb + "\",\"password\":\"" + pwd + "\"}"; + disableCertValidation(); + + String totalstr = null; + try { + URL url = new URL(apiUrl); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestProperty("Content-length", String.valueOf(jStr.length())); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setRequestMethod("GET"); + connection.setDoOutput(true); + connection.setDoInput(true); + DataOutputStream output = new DataOutputStream(connection.getOutputStream()); + output.writeBytes(jStr); + output.close(); + BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + StringBuilder response = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + response.append(line); + } + reader.close(); + + JsonObject input = null; + JsonElement total = null; + JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject(); + if (rootNode.isJsonObject()) { + input = rootNode.getAsJsonObject(); + if (input.isJsonObject()) { + total = input.get("jwt"); + } + } + JsonPrimitive column = total.getAsJsonPrimitive(); + totalstr = column.getAsJsonPrimitive().toString(); + connection.disconnect(); + } catch (Exception e) { + e.printStackTrace(); + } + + return totalstr; + + } + + /** + * Get JWT from CM + *

+ * + * @param apiUrl url to CM auth + * @param username username on CM + * @param pwd password on CM + * @return String jwt + */ + + public static String geAuthToken(String apiUrl) throws Exception + + { + + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + String jStr = "{\"username\":\"" + userName + "\",\"password\":\"" + password + "\"}"; + + disableCertValidation(); + + String totalstr = null; + try { + URL url = new URL(apiUrl); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestProperty("Content-length", String.valueOf(jStr.length())); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setRequestMethod("GET"); + connection.setDoOutput(true); + connection.setDoInput(true); + DataOutputStream output = new DataOutputStream(connection.getOutputStream()); + output.writeBytes(jStr); + output.close(); + BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + StringBuilder response = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + response.append(line); + } + reader.close(); + if (debug) + System.out.println("response " + response); + JsonObject input = null; + JsonElement total = null; + JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject(); + if (rootNode.isJsonObject()) { + input = rootNode.getAsJsonObject(); + if (input.isJsonObject()) { + total = input.get("jwt"); + } + } + JsonPrimitive column = total.getAsJsonPrimitive(); + totalstr = column.getAsJsonPrimitive().toString(); + connection.disconnect(); + } catch (Exception e) { + e.printStackTrace(); + } + + return totalstr; + + } + + public static void disableCertValidation() throws Exception { + TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() { + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + + public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) { + } + + public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) { + } + } }; + + // Install the all-trusting trust manager + try { + SSLContext sc = SSLContext.getInstance("SSL"); + sc.init(null, trustAllCerts, new java.security.SecureRandom()); + HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); + } catch (Exception e) { + e.printStackTrace(); + } + + // Create all-trusting host name verifier + HostnameVerifier allHostsValid = new HostnameVerifier() { + public boolean verify(String hostname, SSLSession session) { + return true; + } + }; + + // Install the all-trusting host verifier + HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid); + } + +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/CustomException.java b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/CustomException.java new file mode 100644 index 00000000..44796974 --- /dev/null +++ b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/CustomException.java @@ -0,0 +1,15 @@ + + + +class CustomException extends Exception { + private int errorCode; + + public CustomException(String message, int errorCode) { + super(message); + this.errorCode = errorCode; + } + + public int getErrorCode() { + return errorCode; + } +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCADPBulkFPETester.java b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCADPBulkFPETester.java new file mode 100644 index 00000000..7beb3ef5 --- /dev/null +++ b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCADPBulkFPETester.java @@ -0,0 +1,525 @@ + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; + +import com.google.cloud.functions.HttpFunction; +import com.google.cloud.functions.HttpRequest; +import com.google.cloud.functions.HttpResponse; +import com.ingrian.security.nae.IngrianProvider; +import com.ingrian.security.nae.AbstractNAECipher; +import com.ingrian.security.nae.FPEParameterAndFormatSpec; +import com.ingrian.security.nae.NAEKey; +import com.ingrian.security.nae.NAESession; +import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder; +import com.ingrian.security.nae.IngrianProvider.Builder; +import com.ingrian.security.nae.NAECipher; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; + +import java.math.BigInteger; +import java.security.spec.AlgorithmParameterSpec; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Logger; + +/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application + * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the + * data so applications or business intelligence tools do not have to change in order to use these columns. +* +* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested +* for all possible data sizes and combinations of encryption algorithms and IV, etc. +* Was tested with CM 2.14 & CADP 8.16.001 +* For more information on CADP see link below. +https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html +* For more information on Snowflake External Functions see link below. +https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp + +Notes: This example uses the CADP bulk API. +Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data +element has Unicode characters then the supported size limit would be 1750 characters + +Size of spec array should be same as the number of elements if user wants to use separate spec values +for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data +index. + * + *@author mwarner + * + */ +public class ThalesGCPSnowCADPBulkFPETester implements HttpFunction { +// @Override + private static byte[][] data; + private static AlgorithmParameterSpec[] spec; + private static Integer[] snowrownbrs; + + private static int BATCHLIMIT = 10000; + private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPBulkFPETester.class.getName()); + private static final Gson gson = new Gson(); + private static final String BADDATATAG = new String ("9999999999999999"); + + public static void main(String[] args) throws Exception { + + ThalesGCPSnowCADPBulkFPETester nw2 = new ThalesGCPSnowCADPBulkFPETester(); + + String requestnull = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"\" ],\r\n" + " [ 1, \"null\" ]\r\n" + + " ]\r\n" + "}"; + + String request = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"pagetest\" ],\r\n" + " [1, \"11\" ],\r\n" + + " [2, \"0\" ],\r\n" + " [ 3, \"life, the universe, and everything\" ]\r\n" + " ]\r\n" + "}"; + + String requestnbr = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"23423453244\" ],\r\n" + + " [1, \"11\" ],\r\n" + " [2, \"000000\" ],\r\n" + " [ 3, \"4523048723948773883\" ]\r\n" + + " ]\r\n" + "}"; + + String protect_nbr102 = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"0001\" ],\r\n" + + " [1, \"0002\" ],\r\n" + " [2, \"0003\" ],\r\n" + " [3, \"0004\" ],\r\n" + + " [4, \"0005\" ],\r\n" + " [5, \"0006\" ],\r\n" + " [6, \"0007\" ],\r\n" + + " [7, \"0008\" ],\r\n" + " [8, \"0009\" ],\r\n" + " [9, \"0010\" ],\r\n" + + " [10, \"0011\" ],\r\n" + " [ 11, \"0012\" ]\r\n" + " ]\r\n" + "}"; + + + String charint = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"2342-345-3244\" ],\r\n" + + " [1, \"1\" ],\r\n" + " [2, \"000-000\" ],\r\n" + + " [ 3, \"4523-048723-948773883\" ]\r\n" + " ]\r\n" + "}"; + + String charintdec = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"1030-236-1715\" ],\r\n" + + " [1, \"17042082183094609\" ],\r\n" + " [2, \"557-917\" ],\r\n" + + " [ 3, \"5078-969877-741002365\" ]\r\n" + " ]\r\n" + "}"; + + //snowflakereturnstring = {"data":[[0,"1030-236-1715"],[1,"17042082183094609"],[2,"557-917"],[3,"5078-969877-741002365"]]} + + String response = null; + nw2.service(protect_nbr102, response); + + } + + public void service(String request, String response) throws Exception { + + String tweakAlgo = null; + String tweakData = null; + String snowflakereturnstring = null; + JsonArray snowflakedata = null; + NAESession session = null; + + Map encryptedErrorMapTotal = new HashMap(); + JsonObject body = null; + int statusCode = 200; + int numberofchunks = 0; + + String keyName = "testfaas"; + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be + // returned when the user above does not have access to the key and if doing a lookup in the userset + // and the user does not exist. If returnciphertextforuserwithnokeyaccess = no then an error will be + // returned to the query, else the results set will provide ciphertext. + // yes/no + String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess"); + boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes"); + + // usersetlookup = should a userset lookup be done on the user from Cloud DB? + // yes/no + String usersetlookup = System.getenv("usersetlookup"); + // usersetidincm = should be the usersetid in CM to query. + String usersetID = System.getenv("usersetidincm"); + // usersetlookupip = this is the IP address to query the userset. Currently it + // is the userset in CM but could be a memcache or other in memory db. + String userSetLookupIP = System.getenv("usersetlookupip"); + boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes"); + String datatype = System.getenv("datatype"); + // mode of operation valid values are : encrypt or decrypt + String mode = System.getenv("mode"); + + int cipherType = 0; + if (mode.equals("encrypt")) + cipherType = Cipher.ENCRYPT_MODE; + else + cipherType = Cipher.DECRYPT_MODE; + + int batchsize = Integer.parseInt(System.getenv("BATCHSIZE")); + + try { + + // This code is only to be used when input data contains user info. + /* + * if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group + * + * boolean founduserinuserset = findUserInUserSet(bigquerysessionUser, userName, password, usersetID, + * userSetLookupIP); // System.out.println("Found User " + founduserinuserset); if (!founduserinuserset) + * throw new CustomException("1001, User Not in User Set", 1001); + * + * } else { usersetlookupbool = false; } + */ + // How many records in a chunk. Testing has indicated point of diminishing returns at 100 or 200, but + // may vary depending on size of data. + + try { + JsonElement requestParsed = gson.fromJson(request, JsonElement.class); + JsonObject requestJson = null; + + if (requestParsed != null && requestParsed.isJsonObject()) { + requestJson = requestParsed.getAsJsonObject(); + } + + if (requestJson != null && requestJson.has("data")) { + snowflakedata = requestJson.getAsJsonArray("data"); + + } + + } catch (JsonParseException e) { + logger.severe("Error parsing JSON: " + e.getMessage()); + } + + System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", + "D:\\product\\Build\\CADP_for_JAVA.properties"); + + session = NAESession.getSession(userName, password.toCharArray()); + NAEKey key = NAEKey.getSecretKey(keyName, session); + int row_number = 0; + + StringBuffer snowflakereturndata = new StringBuffer(); + int numberOfLines = snowflakedata.size(); + int totalRowsLeft = numberOfLines; + + if (batchsize > numberOfLines) + batchsize = numberOfLines; + if (batchsize >= BATCHLIMIT) + batchsize = BATCHLIMIT; + spec = new FPEParameterAndFormatSpec[batchsize]; + data = new byte[batchsize][]; + snowrownbrs = new Integer[batchsize]; + JsonObject bodyObject = new JsonObject(); + JsonArray dataArray = new JsonArray(); + JsonArray innerDataArray = new JsonArray(); + + String algorithm = null; + + FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo) + .build(); + + if (datatype.equals("char")) + algorithm = "FPE/FF1/CARD62"; + else if (datatype.equals("charint")) + algorithm = "FPE/FF1/CARD10"; + else + algorithm = "FPE/FF1/CARD10"; + + AbstractNAECipher thalesCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider"); + int i = 0; + + if (batchsize > numberOfLines) + batchsize = numberOfLines; + if (batchsize >= BATCHLIMIT) + batchsize = BATCHLIMIT; + + spec = new FPEParameterAndFormatSpec[batchsize]; + data = new byte[batchsize][]; + snowrownbrs = new Integer[batchsize]; + String sensitive = null; + thalesCipher.init(cipherType, key, spec[0]); + + // int i = 0; + int count = 0; + boolean newchunk = true; + int dataIndex = 0; + int specIndex = 0; + int snowRowIndex = 0; + // String sensitive = null; + + while (i < numberOfLines) { + int index = 0; + + if (newchunk) { + + if (totalRowsLeft < batchsize) { + spec = new FPEParameterAndFormatSpec[totalRowsLeft]; + data = new byte[totalRowsLeft][]; + snowrownbrs = new Integer[totalRowsLeft]; + } else { + spec = new FPEParameterAndFormatSpec[batchsize]; + data = new byte[batchsize][]; + snowrownbrs = new Integer[batchsize]; + } + newchunk = false; + } + + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + for (int j = 0; j < snowflakerow.size(); j++) { + + if (j == 1) { + + // FPE example + sensitive = checkValid(snowflakerow); + + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + sensitive = BADDATATAG+sensitive; + data[dataIndex++] = sensitive.getBytes(); + // Can not return number since a leading 0 will not work. + // innerDataArray.add(new BigInteger(sensitive)); + } else + data[dataIndex++] = BADDATATAG.getBytes(); + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + data[dataIndex++] = sensitive.getBytes(); + } else { + data[dataIndex++] = sensitive.getBytes(); + } + spec[specIndex++] = param; + + System.out.println("not valid or null"); + } else { + data[dataIndex++] = sensitive.getBytes(); + spec[specIndex++] = param; + } + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + row_number = snowflakecolumn.getAsInt(); + snowrownbrs[snowRowIndex++] = row_number; + + } + + } + + if (count == batchsize - 1) { + + // Map to store exceptions while encryption + Map encryptedErrorMap = new HashMap(); + + byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap); + + for (Map.Entry entry : encryptedErrorMap.entrySet()) { + Integer mkey = entry.getKey(); + String mvalue = entry.getValue(); + encryptedErrorMapTotal.put(mkey, mvalue); + } + + for (int enc = 0; enc < encryptedData.length; enc++) { + + innerDataArray.add(snowrownbrs[enc]); + innerDataArray.add(new String(encryptedData[enc])); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + index++; + + } + + numberofchunks++; + newchunk = true; + count = 0; + dataIndex = 0; + specIndex = 0; + snowRowIndex = 0; + } else + count++; + + totalRowsLeft--; + i++; + } + if (count > 0) { + numberofchunks++; + int index = 0; + Map encryptedErrorMap = new HashMap(); + byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap); + for (int enc = 0; enc < encryptedData.length; enc++) { + innerDataArray.add(snowrownbrs[enc]); + innerDataArray.add(new String(encryptedData[enc])); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + index++; + + } + } + + bodyObject.add("data", dataArray); + String bodyString = bodyObject.toString(); + + snowflakereturnstring = bodyString.toString(); + + } catch ( + + Exception e) { + System.out.println("in exception with " + e.getMessage()); + if (returnciphertextbool) { + if (e.getMessage().contains("1401") + || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) { + + try { + snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, true, null, datatype); + } catch (IllegalBlockSizeException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } catch (BadPaddingException e1) { + // TODO Auto-generated catch block + e1.printStackTrace(); + } + + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } finally { + if (session != null) { + session.closeSession(); + } + } + System.out.println("number of chunks = " + numberofchunks); + System.out.println("snowflakereturnstring = " + snowflakereturnstring); + // response.getWriter().write(snowflakereturnstring); + + } + + public String formatReturnValue(int statusCode) + + { + StringBuffer snowflakereturndatasb = new StringBuffer(); + + snowflakereturndatasb.append("{ \"statusCode\":"); + snowflakereturndatasb.append(statusCode); + snowflakereturndatasb.append(","); + snowflakereturndatasb.append(" \"body\": {"); + snowflakereturndatasb.append(" \"data\": ["); + snowflakereturndatasb.append("] }}"); + System.out.println("in exception with "); + return snowflakereturndatasb.toString(); + } + + public String checkValid(JsonArray snowrow) { + String inputdata = null; + String notvalid = "notvalid"; + if (snowrow != null && snowrow.size() > 0) { + JsonElement element = snowrow.get(1); + if (element != null && !element.isJsonNull()) { + inputdata = element.getAsString(); + if (inputdata.isEmpty() || inputdata.length() < 2) { + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("Sensitive data is null or empty."); + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("bigquerytrow is null or empty."); + inputdata = notvalid + inputdata; + } + + return inputdata; + + } + + + public String formatReturnValue(int statusCode, JsonArray snowflakedata, boolean error, Cipher thalesCipher, + String datatype) throws IllegalBlockSizeException, BadPaddingException { + int row_number = 0; + + String encdata = null; + String sensitive = null; + + JsonObject bodyObject = new JsonObject(); + JsonArray dataArray = new JsonArray(); + JsonArray innerDataArray = new JsonArray(); + + for (int i = 0; i < snowflakedata.size(); i++) { + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + for (int j = 0; j < snowflakerow.size(); j++) { + if (j == 1) { + sensitive = checkValid(snowflakerow); + + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + System.out.println("adding null not charint or nbr"); + sensitive = sensitive.replace("notvalid", ""); + innerDataArray.add(sensitive); + // Can not return number since a leading 0 will not work. + // innerDataArray.add(new BigInteger(sensitive)); + } else + innerDataArray.add(BADDATATAG); + + System.out.println("datatype charint or nbr adding big int 9999999999999999"); + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + innerDataArray.add(""); + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + innerDataArray.add(sensitive); + } else { + innerDataArray.add(sensitive); + } + + } else { + if (!error) { + byte[] outbuf = thalesCipher.doFinal(sensitive.getBytes()); + encdata = new String(outbuf); + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + innerDataArray.add(encdata); + // innerDataArray.add(new BigInteger(encdata)); + } else { + innerDataArray.add(encdata); + } + } else { + encdata = sensitive; + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + innerDataArray.add(encdata); + // innerDataArray.add(new BigInteger(encdata)); + } else { + innerDataArray.add(encdata); + } + } + } + + // innerDataArray.add(encdata); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + row_number = snowflakecolumn.getAsInt(); + innerDataArray.add(row_number); + } + } + } + + bodyObject.add("data", dataArray); + String bodyString = bodyObject.toString(); + + System.out.println("bodystr " + bodyString); + + return bodyString; + + } + + public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID, + String userSetLookupIP) throws Exception { + + CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP); + + String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd); + String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken); + + boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken); + + return founduserinuserset; + + } + + @Override + public void service(HttpRequest request, HttpResponse response) throws Exception { + // TODO Auto-generated method stub + + } +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCADPFPETester.java b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCADPFPETester.java new file mode 100644 index 00000000..d1bb677e --- /dev/null +++ b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCADPFPETester.java @@ -0,0 +1,337 @@ + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; + +import com.google.cloud.functions.HttpFunction; +import com.google.cloud.functions.HttpRequest; +import com.google.cloud.functions.HttpResponse; +import com.ingrian.security.nae.IngrianProvider; +import com.ingrian.security.nae.FPECharset; +import com.ingrian.security.nae.FPEParameterAndFormatSpec; +import com.ingrian.security.nae.NAEKey; +import com.ingrian.security.nae.NAESession; +import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder; +import com.ingrian.security.nae.IngrianProvider.Builder; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; + +import java.math.BigInteger; +import java.util.logging.Logger; + +/* This test app to test the logic for a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application + * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the + * data so applications or business intelligence tools do not have to change in order to use these columns. +* +* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested +* for all possible data sizes and combinations of encryption algorithms and IV, etc. +* Was tested with CM 2.14 & CADP 8.16 +* For more information on CADP see link below. +https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html +* For more information on Snowflake External Functions see link below. +https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp + * + *@author mwarner + * + */ +public class ThalesGCPSnowCADPFPETester implements HttpFunction { +// @Override + + private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPFPETester.class.getName()); + private static final Gson gson = new Gson(); + private static final String BADDATATAG = new String ("9999999999999999"); + + public static void main(String[] args) throws Exception { + + ThalesGCPSnowCADPFPETester nw2 = new ThalesGCPSnowCADPFPETester(); + + String requestnull = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"\" ],\r\n" + " [ 1, \"null\" ]\r\n" + + " ]\r\n" + "}"; + + String request = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"pagetest\" ],\r\n" + " [1, \"11\" ],\r\n" + + " [2, \"0\" ],\r\n" + " [ 3, \"life, the universe, and everything\" ]\r\n" + " ]\r\n" + "}"; + + String requestnbr = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"342342348834\" ],\r\n" + + " [1, \"56\" ],\r\n" + " [2, \"0\" ],\r\n" + " [ 3, \"452345778833333\" ]\r\n" + + " ]\r\n" + "}"; + + String requestnbrinvalid = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"342342348834\" ],\r\n" + + " [1, \"null\" ],\r\n" + " [2, \"0\" ],\r\n" + " [ 3, \"452345778833333\" ]\r\n" + + " ]\r\n" + "}"; + + String response = null; + nw2.service(requestnbr, response); + + } + + public void service(String request, String response) throws Exception { + + String encdata = ""; + String tweakAlgo = null; + String tweakData = null; + String snowflakereturnstring = null; + JsonArray snowflakedata = null; + int statusCode = 200; + + NAESession session = null; + String keyName = "testfaas"; + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + + // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned + // when the user above does not have access to the key and if doing a + // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no + // then an error will be returned to the query, else the results set will provide ciphertext. + String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess"); + // yes,no + boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes"); + // usersetlookup = should a userset lookup be done on the user from Cloud DB + // yes,no + String usersetlookup = System.getenv("usersetlookup"); + // usersetidincm = should be the usersetid in CM to query. + String usersetID = System.getenv("usersetidincm"); + // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be + // a memcache or other in memory db. + String userSetLookupIP = System.getenv("usersetlookupip"); + boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes"); + String datatype = System.getenv("datatype"); + // mode of operation valid values are : encrypt or decrypt + String mode = System.getenv("mode"); + + int cipherType = 0; + if (mode.equals("encrypt")) + cipherType = Cipher.ENCRYPT_MODE; + else + cipherType = Cipher.DECRYPT_MODE; + + try { + + // This code is only to be used when input data contains user info. + /* + * if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group + * + * boolean founduserinuserset = findUserInUserSet(bigquerysessionUser, userName, password, usersetID, + * userSetLookupIP); // System.out.println("Found User " + founduserinuserset); if (!founduserinuserset) + * throw new CustomException("1001, User Not in User Set", 1001); + * + * } else { usersetlookupbool = false; } + */ + + try { + JsonElement requestParsed = gson.fromJson(request, JsonElement.class); + JsonObject requestJson = null; + + if (requestParsed != null && requestParsed.isJsonObject()) { + requestJson = requestParsed.getAsJsonObject(); + } + + if (requestJson != null && requestJson.has("data")) { + snowflakedata = requestJson.getAsJsonArray("data"); + + } + + } catch (JsonParseException e) { + logger.severe("Error parsing JSON: " + e.getMessage()); + } + + /* + * System.setProperty( "com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", + * "CADP_for_JAVA.properties"); IngrianProvider builder = new Builder().addConfigFileInputStream( + * getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")). build(); + */ + + System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", + "D:\\product\\Build\\CADP_for_JAVA.properties"); + + session = NAESession.getSession(userName, password.toCharArray()); + NAEKey key = NAEKey.getSecretKey(keyName, session); + int row_number = 0; + + String algorithm = null; + + FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo).build(); + if (datatype.equals("char")) + { + algorithm = "FPE/FF1/CARD62"; + } + else { + algorithm = "FPE/FF1/CARD10"; + } + + Cipher thalesCipher = Cipher.getInstance(algorithm, "IngrianProvider"); + // initialize cipher to encrypt. + thalesCipher.init(cipherType, key, param); + + snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, false, thalesCipher, datatype); + + } catch (Exception e) { + + System.out.println("in exception with " + e.getMessage()); + if (returnciphertextbool) { + if (e.getMessage().contains("1401") + || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) { + + snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, true, null, datatype); + + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + + } finally { + if (session != null) { + session.closeSession(); + } + } + System.out.println(snowflakereturnstring); + // response.getWriter().write(snowflakereturnstring); + + } + + public String checkValid(JsonArray snowrow) { + String inputdata = null; + String notvalid = "notvalid"; + if (snowrow != null && snowrow.size() > 0) { + JsonElement element = snowrow.get(1); + if (element != null && !element.isJsonNull()) { + inputdata = element.getAsString(); + if (inputdata.isEmpty() || inputdata.length() < 2) { + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("Sensitive data is null or empty."); + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("bigquerytrow is null or empty."); + inputdata = notvalid + inputdata; + } + + return inputdata; + + } + + public String formatReturnValue(int statusCode) + + { + StringBuffer snowflakereturndatasb = new StringBuffer(); + + snowflakereturndatasb.append("{ \"statusCode\":"); + snowflakereturndatasb.append(statusCode); + snowflakereturndatasb.append(","); + snowflakereturndatasb.append(" \"body\": {"); + snowflakereturndatasb.append(" \"data\": ["); + snowflakereturndatasb.append("] }}"); + System.out.println("in exception with "); + return snowflakereturndatasb.toString(); + } + + public String formatReturnValue(int statusCode, JsonArray snowflakedata, boolean error, Cipher thalesCipher, + String datatype) throws IllegalBlockSizeException, BadPaddingException { + int row_number = 0; + + String encdata = null; + String sensitive = null; + + JsonObject bodyObject = new JsonObject(); + JsonArray dataArray = new JsonArray(); + JsonArray innerDataArray = new JsonArray(); + + for (int i = 0; i < snowflakedata.size(); i++) { + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + for (int j = 0; j < snowflakerow.size(); j++) { + if (j == 1) { + sensitive = checkValid(snowflakerow); + + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + System.out.println("adding null not charint or nbr"); + sensitive = sensitive.replace("notvalid", ""); + innerDataArray.add(sensitive); + // Can not return number since a leading 0 will not work. + // innerDataArray.add(new BigInteger(sensitive)); + } else + innerDataArray.add(BADDATATAG); + + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + innerDataArray.add(""); + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + innerDataArray.add(sensitive); + } else { + innerDataArray.add(sensitive); + } + + } else { + if (!error) { + byte[] outbuf = thalesCipher.doFinal(sensitive.getBytes()); + encdata = new String(outbuf); + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + innerDataArray.add(encdata); + // innerDataArray.add(new BigInteger(encdata)); + } else { + innerDataArray.add(encdata); + } + } else { + encdata = sensitive; + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + innerDataArray.add(encdata); + // innerDataArray.add(new BigInteger(encdata)); + } else { + innerDataArray.add(encdata); + } + } + } + + // innerDataArray.add(encdata); + dataArray.add(innerDataArray); + innerDataArray = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + row_number = snowflakecolumn.getAsInt(); + innerDataArray.add(row_number); + } + } + } + + bodyObject.add("data", dataArray); + String bodyString = bodyObject.toString(); + + System.out.println("bodystr " + bodyString); + + return bodyString; + + } + + public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID, + String userSetLookupIP) throws Exception { + + CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP); + + String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd); + String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken); + + boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken); + + return founduserinuserset; + + } + + @Override + public void service(HttpRequest request, HttpResponse response) throws Exception { + // TODO Auto-generated method stub + + } +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCRDPBulkFPETester.java b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCRDPBulkFPETester.java new file mode 100644 index 00000000..b5eef4f7 --- /dev/null +++ b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCRDPBulkFPETester.java @@ -0,0 +1,568 @@ + +import com.google.cloud.functions.HttpFunction; +import com.google.cloud.functions.HttpRequest; +import com.google.cloud.functions.HttpResponse; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; + +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +import java.util.HashMap; +import java.util.Map; + +import java.util.logging.Logger; + +/* This test app to test the logic for a Snowflake Database User Defined Function(UDF). + * It is an example of how to use Thales CipherTrust REST Application Dataprotection (CRDP) + * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the + * data so applications or business intelligence tools do not have to change in order to use these columns. +* +* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested +* for all possible data sizes and combinations of encryption algorithms and IV, etc. +* Was tested with CM 2.14 and CRDP 1.0 +* For more information on CRDP see link below. +https://thalesdocs.com/ctp/con/crdp/latest/index.html +* For more information on Snowflake External Functions see link below. +https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp + * + *@author mwarner + * + */ +public class ThalesGCPSnowCRDPBulkFPETester implements HttpFunction { +// @Override + + private static final Gson gson = new Gson(); + private static final String BADDATATAG = new String("9999999999999999"); + private static int BATCHLIMIT = 10000; + + private static final String REVEALRETURNTAG = new String("data"); + private static final String PROTECTRETURNTAG = new String("protected_data"); + + public static void main(String[] args) throws Exception { + + ThalesGCPSnowCRDPBulkFPETester nw2 = new ThalesGCPSnowCRDPBulkFPETester(); + + String protectnull = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"null\" ],\r\n" + " [1, \"1\" ],\r\n" + + " [ 2, \"\" ]\r\n" + " ]\r\n" + "}"; + + String revealnull = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"9500405729277875\" ],\r\n" + + " [1, \"83724488316877864\" ],\r\n" + " [ 2, \"9500405729277875\" ]\r\n" + " ]\r\n" + "}"; + // 1002001 {"data":[[0,"9500405729277875"],[1,"83724488316877864"],[2,"9500405729277875"]]} + String protect = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"pagetest\" ],\r\n" + + " [1, \"thisismoredata\" ],\r\n" + " [2, \"8989977666\" ],\r\n" + + " [ 3, \"life, the universe, and everything\" ]\r\n" + " ]\r\n" + "}"; + + String protect_nbr = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"234242344\" ],\r\n" + + " [1, \"6578-5545-7567\" ],\r\n" + " [2, \"3434\" ],\r\n" + " [ 3, \"333\" ]\r\n" + + " ]\r\n" + "}"; + + String reveal_ext_nbr = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"652425219\" ],\r\n" + + " [1, \"6684-5327-7937\" ],\r\n" + " [2, \"5295\" ],\r\n" + " [ 3, \"856\" ]\r\n" + + " ]\r\n" + "}"; + + String protect_nbr102 = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"0001\" ],\r\n" + + " [1, \"0002\" ],\r\n" + " [2, \"0003\" ],\r\n" + " [3, \"0004\" ],\r\n" + + " [4, \"0005\" ],\r\n" + " [5, \"0006\" ],\r\n" + " [6, \"0007\" ],\r\n" + + " [7, \"0008\" ],\r\n" + " [8, \"0009\" ],\r\n" + " [9, \"0010\" ],\r\n" + + " [10, \"0011\" ],\r\n" + " [ 11, \"0012\" ]\r\n" + " ]\r\n" + "}"; + + String reveal_ext_alpha = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"wSf2fghX\" ],\r\n" + + " [1, \"7nGV82KYDX001F\" ],\r\n" + " [2, \"GsrAUKzfje\" ],\r\n" + + " [ 3, \"Qqce, QBv d7ErVAPM, teq ZD6sjHPecT\" ]\r\n" + " ]\r\n" + "}"; + + // String reveal_ext_nbr = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"073675208\" ],\r\n" + // + " [1, \"5336-6160-6623\" ],\r\n" + " [2, \"1348\" ],\r\n" + " [ 3, \"054\" ]\r\n" + // + " ]\r\n" + "}"; + +//temp{"data":[[0,"073675208"],[1,"5336-6160-6623"],[2,"1348"],[3,"054"]]} + + String response = null; + nw2.service(protect_nbr102, response); + + } + + public void service(String request, String response) throws Exception { + Map bqErrorMap = new HashMap(); + String encdata = ""; + int error_count = 0; + int statusCode = 200; + + JsonObject result = new JsonObject(); + JsonArray replies = new JsonArray(); + JsonArray rownbranddata = new JsonArray(); + + String keyName = "testfaas"; + String crdpip = System.getenv("CRDPIP"); + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + + // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned + // when the user above does not have access to the key and if doing a + // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no + // then an error will be returned to the query, else the results set will provide ciphertext. + String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess"); + // yes,no + boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes"); + // usersetlookup = should a userset lookup be done on the user from Cloud DB + // yes,no + String usersetlookup = System.getenv("usersetlookup"); + // usersetidincm = should be the usersetid in CM to query. + String usersetID = System.getenv("usersetidincm"); + // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be + // a memcache or other in memory db. + String userSetLookupIP = System.getenv("usersetlookupip"); + boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes"); + String keymetadatalocation = System.getenv("keymetadatalocation"); + String external_version_from_ext_source = System.getenv("keymetadata"); + String protection_profile = System.getenv("protection_profile"); + String mode = System.getenv("mode"); + String datatype = System.getenv("datatype"); + + String inputDataKey = null; + String outputDataKey = null; + String protectedData = null; + String externalkeymetadata = null; + String jsonBody = null; + String jsonTagForProtectReveal = null; + + boolean bad_data = false; + if (mode.equals("protectbulk")) { + inputDataKey = "data_array"; + outputDataKey = "protected_data_array"; + jsonTagForProtectReveal = PROTECTRETURNTAG; + } else { + inputDataKey = "protected_data_array"; + outputDataKey = "data_array"; + jsonTagForProtectReveal = REVEALRETURNTAG; + } + + String snowflakeuser = "snowflakeuser"; + JsonArray snowflakedata = null; + String snowflakereturnstring = null; + Response crdp_response = null; + try { + + // This code is only to be used when input data contains user info. + /* + * if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group + * + * boolean founduserinuserset = findUserInUserSet(bigquerysessionUser, userName, password, usersetID, + * userSetLookupIP); // System.out.println("Found User " + founduserinuserset); if (!founduserinuserset) + * throw new CustomException("1001, User Not in User Set", 1001); + * + * } else { usersetlookupbool = false; } + */ + + try { + JsonElement requestParsed = gson.fromJson(request, JsonElement.class); + JsonObject requestJson = null; + + if (requestParsed != null && requestParsed.isJsonObject()) { + requestJson = requestParsed.getAsJsonObject(); + } + + if (requestJson != null && requestJson.has("data")) { + snowflakedata = requestJson.getAsJsonArray("data"); + + } + + } catch (JsonParseException e) { + System.out.println("Error parsing JSON: " + e.getMessage()); + } + + int row_number = 0; + + // Serialization + + int numberofchunks = 0; + StringBuffer protection_policy_buff = new StringBuffer(); + String notvalid = "notvalid"; + // StringBuffer snowflakereturndata = new StringBuffer(); + + int numberOfLines = snowflakedata.size(); + int totalRowsLeft = numberOfLines; + // int batchsize = Integer.parseInt(System.getenv("BATCHSIZE")); + int batchsize = 10; + if (batchsize > numberOfLines) + batchsize = numberOfLines; + if (batchsize >= BATCHLIMIT) + batchsize = BATCHLIMIT; + // Serialization + + String crdpjsonBody = null; + // String external_version_from_ext_source = "1004001"; + // String external_version_from_ext_source = "1001001"; + + int i = 0; + int count = 0; + int totalcount = 0; + int dataIndex = 0; // assumes index from snowflake will always be sequential. + JsonObject crdp_payload = new JsonObject(); + String sensitive = null; + JsonArray crdp_payload_array = new JsonArray(); + + OkHttpClient client = new OkHttpClient().newBuilder().build(); + MediaType mediaType = MediaType.parse("application/json"); + String urlStr = "http://" + crdpip + ":8090/v1/" + mode; + + while (i < numberOfLines) { + + for (int b = 0; b < batchsize && b < totalRowsLeft; b++) { + + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + sensitive = checkValid(snowflakerow); + + + protection_profile = protection_profile.trim(); + // Format the output + String formattedElement = String.format("\"protection_policy_name\" : \"%s\"", protection_profile); + protection_policy_buff.append(formattedElement); + protection_policy_buff.append(","); + + if (mode.equals("protectbulk")) { + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + // System.out.println("adding null not charint or nbr"); + sensitive = sensitive.replace("notvalid", ""); + sensitive = BADDATATAG + sensitive; + } else + sensitive = BADDATATAG; + + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + + } else if (sensitive.contains("notvalid")) { + // sensitive = sensitive.replace("notvalid", ""); + + } + encdata = sensitive; + + } + crdp_payload_array.add(sensitive); + } else { + JsonObject protectedDataObject = new JsonObject(); + protectedDataObject.addProperty(PROTECTRETURNTAG, sensitive); + if (keymetadatalocation.equalsIgnoreCase("external")) { + protectedDataObject.addProperty("external_version", external_version_from_ext_source); + } + crdp_payload_array.add(protectedDataObject); + + } + + + if (count == batchsize - 1) { + crdp_payload.add(inputDataKey, crdp_payload_array); + String inputdataarray = null; + if (mode.equals("revealbulk")) { + crdp_payload.addProperty("username", snowflakeuser); + inputdataarray = crdp_payload.toString(); + protection_policy_buff.append(inputdataarray); + jsonBody = protection_policy_buff.toString(); + jsonBody = jsonBody.replaceFirst("\\{", " "); + + } else { + inputdataarray = crdp_payload.toString(); + protection_policy_buff.append(inputdataarray); + inputdataarray = protection_policy_buff.toString(); + jsonBody = inputdataarray.replace("{", " "); + } + jsonBody = "{" + jsonBody; + + RequestBody body = RequestBody.create(mediaType, jsonBody); + + Request crdp_request = new Request.Builder().url(urlStr).method("POST", body) + .addHeader("Content-Type", "application/json").build(); + crdp_response = client.newCall(crdp_request).execute(); + String crdpreturnstr = null; + if (crdp_response.isSuccessful()) { + // Parse JSON response + String responseBody = crdp_response.body().string(); + JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class); + JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey); + + String status = jsonObject.get("status").getAsString(); + int success_count = jsonObject.get("success_count").getAsInt(); + error_count = jsonObject.get("error_count").getAsInt(); + if (error_count > 0) + System.out.println("errors " + error_count); + + for (JsonElement element : protectedDataArray) { + + JsonObject protectedDataObject = element.getAsJsonObject(); + if (protectedDataObject.has(jsonTagForProtectReveal)) { + + protectedData = protectedDataObject.get(jsonTagForProtectReveal).getAsString(); + System.out.println(protectedData); + + rownbranddata.add(dataIndex); + rownbranddata.add(new String(protectedData)); + replies.add(rownbranddata); + rownbranddata = new JsonArray(); + if (mode.equals("protectbulk")) { + if (keymetadatalocation.equalsIgnoreCase("external") && mode.equalsIgnoreCase("protect")) { + externalkeymetadata = protectedDataObject.get("external_version") + .getAsString(); + System.out.println("Protected Data ext key metadata need to store this: " + + externalkeymetadata); + + } + } + } else if (protectedDataObject.has("error_message")) { + String errorMessage = protectedDataObject.get("error_message").getAsString(); + System.out.println("error_message: " + errorMessage); + bqErrorMap.put(i, errorMessage); + bad_data = true; + } else + System.out.println("unexpected json value from results: "); + dataIndex++; + + } + + crdp_payload_array = new JsonArray(); + protection_policy_buff = new StringBuffer(); + numberofchunks++; + totalcount = totalcount + count; + count = 0; + } else {// throw error.... + System.err.println("Request failed with status code: " + crdp_response.code()); + throw new CustomException("1010, Unexpected Error ", 1010); + } + + } else { + count++; + } + totalRowsLeft--; + i++; + } + } + if (count > 0) { + crdp_payload.add(inputDataKey, crdp_payload_array); + String inputdataarray = null; + if (mode.equals("revealbulk")) { + crdp_payload.addProperty("username", snowflakeuser); + inputdataarray = crdp_payload.toString(); + protection_policy_buff.append(inputdataarray); + jsonBody = protection_policy_buff.toString(); + jsonBody = jsonBody.replaceFirst("\\{", " "); + + } else { + inputdataarray = crdp_payload.toString(); + protection_policy_buff.append(inputdataarray); + inputdataarray = protection_policy_buff.toString(); + jsonBody = inputdataarray.replace("{", " "); + } + jsonBody = "{" + jsonBody; + + RequestBody body = RequestBody.create(mediaType, jsonBody); + + Request crdp_request = new Request.Builder().url(urlStr).method("POST", body) + .addHeader("Content-Type", "application/json").build(); + crdp_response = client.newCall(crdp_request).execute(); + String crdpreturnstr = null; + if (crdp_response.isSuccessful()) { + // Parse JSON response + String responseBody = crdp_response.body().string(); + JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class); + JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey); + + String status = jsonObject.get("status").getAsString(); + int success_count = jsonObject.get("success_count").getAsInt(); + error_count = jsonObject.get("error_count").getAsInt(); + if (error_count > 0) + System.out.println("errors " + error_count); + + for (JsonElement element : protectedDataArray) { + + JsonObject protectedDataObject = element.getAsJsonObject(); + if (protectedDataObject.has(jsonTagForProtectReveal)) { + + protectedData = protectedDataObject.get(jsonTagForProtectReveal).getAsString(); + System.out.println(protectedData); + + rownbranddata.add(dataIndex); + rownbranddata.add(new String(protectedData)); + replies.add(rownbranddata); + rownbranddata = new JsonArray(); + if (mode.equals("protectbulk")) { + if (keymetadatalocation.equalsIgnoreCase("external") && mode.equalsIgnoreCase("protect")) { + externalkeymetadata = protectedDataObject.get("external_version").getAsString(); + System.out.println("Protected Data ext key metadata need to store this: " + + externalkeymetadata); + + } + } + } else if (protectedDataObject.has("error_message")) { + String errorMessage = protectedDataObject.get("error_message").getAsString(); + System.out.println("error_message: " + errorMessage); + bqErrorMap.put(i, errorMessage); + bad_data = true; + } else + System.out.println("unexpected json value from results: "); + dataIndex++; + + } + + crdp_payload_array = new JsonArray(); + protection_policy_buff = new StringBuffer(); + numberofchunks++; + totalcount = totalcount + count; + count = 0; + + } else { + System.err.println("Request failed with status code: " + crdp_response.code()); + } + } + crdp_response.close(); + System.out.println("total chuncks " + numberofchunks); + + result.add("data", replies); + + snowflakereturnstring = result.toString(); + + } catch (Exception e) { + System.out.println("in exception with " + e.getMessage()); + snowflakereturnstring = "exception "; + if (returnciphertextbool) { + if (e.getMessage().contains("1401") + || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) { + + result = new JsonObject(); + replies = new JsonArray(); + rownbranddata = new JsonArray(); + + for (int i = 0; i < snowflakedata.size(); i++) { + + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + for (int j = 0; j < snowflakerow.size(); j++) { + if (j == 1) { + // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); + // FPE example + String sensitive = checkValid(snowflakerow); + + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + } else + sensitive = BADDATATAG; + + } else if (sensitive.equalsIgnoreCase("null") + || sensitive.equalsIgnoreCase("notvalid")) { + + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + + } + encdata = sensitive; + + } else { + System.out.println("normal number data" + sensitive); + } + rownbranddata.add(sensitive); + replies.add(rownbranddata); + rownbranddata = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + int row_number = snowflakecolumn.getAsInt(); + rownbranddata.add(row_number); + } + } + } + + result.add("data", replies); + JsonObject inputJsonObject = new JsonObject(); + String bodyString = result.toString(); + inputJsonObject.addProperty("statusCode", 200); + inputJsonObject.addProperty("body", bodyString); + + snowflakereturnstring = inputJsonObject.toString(); + System.out.println(" new data " + snowflakereturnstring); + + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + + } finally { + + } + System.out.println(snowflakereturnstring); + // response.getWriter().write(snowflakereturnstring); + + } + + public String formatReturnValue(int statusCode) + + { + StringBuffer snowflakereturndatasb = new StringBuffer(); + + snowflakereturndatasb.append("{ \"statusCode\":"); + snowflakereturndatasb.append(statusCode); + snowflakereturndatasb.append(","); + snowflakereturndatasb.append(" \"body\": {"); + snowflakereturndatasb.append(" \"data\": ["); + snowflakereturndatasb.append("] }}"); + System.out.println("in exception with "); + return snowflakereturndatasb.toString(); + } + + public String checkValid(JsonArray snowrow) { + String inputdata = null; + String notvalid = "notvalid"; + if (snowrow != null && snowrow.size() > 0) { + JsonElement element = snowrow.get(1); + if (element != null && !element.isJsonNull()) { + inputdata = element.getAsString(); + if (inputdata.isEmpty() || inputdata.length() < 2) { + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("Sensitive data is null or empty."); + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("bigquerytrow is null or empty."); + inputdata = notvalid + inputdata; + } + + return inputdata; + + } + + public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID, + String userSetLookupIP) throws Exception { + + CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP); + + String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd); + String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken); + + boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken); + + return founduserinuserset; + + } + + @Override + public void service(HttpRequest request, HttpResponse response) throws Exception { + // TODO Auto-generated method stub + + } +} \ No newline at end of file diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCRDPFPETester.java b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCRDPFPETester.java new file mode 100644 index 00000000..05f7a3d8 --- /dev/null +++ b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCRDPFPETester.java @@ -0,0 +1,415 @@ + +import com.google.cloud.functions.HttpFunction; +import com.google.cloud.functions.HttpRequest; +import com.google.cloud.functions.HttpResponse; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonPrimitive; + +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +import java.math.BigInteger; +import java.util.HashMap; +import java.util.Map; + +import java.util.logging.Logger; + +/* This test app to test the logic for a Snowflake Database User Defined Function(UDF). + * It is an example of how to use Thales CipherTrust REST Application Dataprotection (CRDP) + * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the + * data so applications or business intelligence tools do not have to change in order to use these columns. +* +* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested +* for all possible data sizes and combinations of encryption algorithms and IV, etc. +* Was tested with CM 2.14 and CRDP 1.0 +* For more information on CRDP see link below. +https://thalesdocs.com/ctp/con/crdp/latest/index.html +* For more information on Snowflake External Functions see link below. +https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp + * + *@author mwarner + * + */ +public class ThalesGCPSnowCRDPFPETester implements HttpFunction { +// @Override + +// @Override + + private static final Gson gson = new Gson(); + private static final String BADDATATAG = new String("9999999999999999"); + + private static final String REVEALRETURNTAG = new String("data"); + private static final String PROTECTRETURNTAG = new String("protected_data"); + + public static void main(String[] args) throws Exception { + + ThalesGCPSnowCRDPFPETester nw2 = new ThalesGCPSnowCRDPFPETester(); + + String protectnull = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"\" ],\r\n" + + " [1, \"somemoredatat\" ],\r\n" + " [ 2, \"null\" ]\r\n" + " ]\r\n" + "}"; + + String protect = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"pagetest\" ],\r\n" + + " [1, \"null\" ],\r\n" + " [2, \"0\" ],\r\n" + + " [ 3, \"life, the universe, and everything\" ]\r\n" + " ]\r\n" + "}"; + + String reveal_ext_alpha = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"wSf2fghX\" ],\r\n" + + " [1, \"xI\" ],\r\n" + " [2, \"notvalid0\" ],\r\n" + + " [ 3, \"Qqce, QBv d7ErVAPM, teq ZD6sjHPecT\" ]\r\n" + " ]\r\n" + "}"; + + String reveal_ext_nbr = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"730712653\" ],\r\n" + + " [1, \"4108-531\" ],\r\n" + " [2, \"0\" ],\r\n" + " [ 3, \"9999999999999999\" ]\r\n" + + " ]\r\n" + "}"; + + String protectnbr = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"345245667\" ],\r\n" + + " [1, \"4578-094\" ],\r\n" + " [2, \"0\" ],\r\n" + " [ 3, \"null\" ]\r\n" + " ]\r\n" + + "}"; + + // "data":[[0,"730712653"],[1,"4108-531"],[2,"0"],[3,"9999999999999999"]]} + String response = null; + nw2.service(protectnbr, response); + + } + + public void service(String request, String response) throws Exception { + Map snowErrorMap = new HashMap(); + String encdata = ""; + String snowflakereturnstring = null; + JsonArray snowflakedata = null; + int statusCode = 200; + + String keyName = "testfaas"; + String crdpip = System.getenv("CRDPIP"); + String userName = System.getenv("CMUSER"); + String password = System.getenv("CMPWD"); + + // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned + // when the user above does not have access to the key and if doing a + // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no + // then an error will be returned to the query, else the results set will provide ciphertext. + String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess"); + // yes,no + boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes"); + // usersetlookup = should a userset lookup be done on the user from Cloud DB + // yes,no + String usersetlookup = System.getenv("usersetlookup"); + // usersetidincm = should be the usersetid in CM to query. + String usersetID = System.getenv("usersetidincm"); + // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be + // a memcache or other in memory db. + String userSetLookupIP = System.getenv("usersetlookupip"); + boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes"); + String keymetadatalocation = System.getenv("keymetadatalocation"); + String external_version_from_ext_source = System.getenv("keymetadata"); + String protection_profile = System.getenv("protection_profile"); + String mode = System.getenv("mode"); + String datatype = System.getenv("datatype"); + String dataKey = null; + String jsonTagForProtectReveal = null; + + boolean bad_data = false; + if (mode.equals("protect")) { + dataKey = "data"; + jsonTagForProtectReveal = PROTECTRETURNTAG; + } else { + dataKey = "protected_data"; + jsonTagForProtectReveal = REVEALRETURNTAG; + } + + String snowflakeuser = "snowflakeuser"; + + JsonObject result = new JsonObject(); + JsonArray replies = new JsonArray(); + JsonArray rownbranddata = new JsonArray(); + + try { + + // This code is only to be used when input data contains user info. + /* + * if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group + * + * boolean founduserinuserset = findUserInUserSet(bigquerysessionUser, userName, password, usersetID, + * userSetLookupIP); // System.out.println("Found User " + founduserinuserset); if (!founduserinuserset) + * throw new CustomException("1001, User Not in User Set", 1001); + * + * } else { usersetlookupbool = false; } + */ + + try { + JsonElement requestParsed = gson.fromJson(request, JsonElement.class); + JsonObject requestJson = null; + + if (requestParsed != null && requestParsed.isJsonObject()) { + requestJson = requestParsed.getAsJsonObject(); + } + + if (requestJson != null && requestJson.has("data")) { + snowflakedata = requestJson.getAsJsonArray("data"); + + } + + } catch (JsonParseException e) { + System.out.println("Error parsing JSON: " + e.getMessage()); + + } + + int row_number = 0; + + String protectedData = null; + String externalkeymetadata = null; + String crdpjsonBody = null; + // String external_version_from_ext_source = "1004001"; + // String external_version_from_ext_source = "1001001"; + + JsonObject crdp_payload = new JsonObject(); + + crdp_payload.addProperty("protection_policy_name", protection_profile); + String sensitive = null; + OkHttpClient client = new OkHttpClient().newBuilder().build(); + MediaType mediaType = MediaType.parse("application/json"); + String urlStr = "http://" + crdpip + ":8090/v1/" + mode; + + // encrypt data + for (int i = 0; i < snowflakedata.size(); i++) { + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + for (int j = 0; j < snowflakerow.size(); j++) { + + if (j == 1) { + // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); + // FPE example + sensitive = checkValid(snowflakerow); + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + } else + sensitive = BADDATATAG; + + } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) { + + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + + } + encdata = sensitive; + + } else { + + crdp_payload.addProperty(dataKey, sensitive); + if (mode.equals("reveal")) { + crdp_payload.addProperty("username", snowflakeuser); + if (keymetadatalocation.equalsIgnoreCase("external")) { + crdp_payload.addProperty("external_version", external_version_from_ext_source); + } + + } + crdpjsonBody = crdp_payload.toString(); + System.out.println(crdpjsonBody); + RequestBody body = RequestBody.create(mediaType, crdpjsonBody); + + Request crdp_request = new Request.Builder() + // .url("http://192.168.159.143:8090/v1/protect").method("POST", body) + .url(urlStr).method("POST", body).addHeader("Content-Type", "application/json") + .build(); + Response crdp_response = client.newCall(crdp_request).execute(); + + if (crdp_response.isSuccessful()) { + + // Parse JSON response + String responseBody = crdp_response.body().string(); + Gson gson = new Gson(); + JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class); + + if (jsonObject.has(jsonTagForProtectReveal)) { + protectedData = jsonObject.get(jsonTagForProtectReveal).getAsString(); + if (keymetadatalocation.equalsIgnoreCase("external") && mode.equalsIgnoreCase("protect")) { + externalkeymetadata = jsonObject.get("external_version").getAsString(); + System.out.println("Protected Data ext key metadata need to store this: " + + externalkeymetadata); + } + } else if (jsonObject.has("error_message")) { + String errorMessage = jsonObject.get("error_message").getAsString(); + System.out.println("error_message: " + errorMessage); + snowErrorMap.put(i, errorMessage); + bad_data = true; + } else + System.out.println("unexpected json value from results: "); + + System.out.println("Protected Data: " + protectedData); + + } else { + System.err.println("Request failed with status code: " + crdp_response.code()); + } + + crdp_response.close(); + + encdata = protectedData; + + } + + rownbranddata.add(encdata); + replies.add(rownbranddata); + rownbranddata = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + row_number = snowflakecolumn.getAsInt(); + rownbranddata.add(row_number); + + } + + } + + } + + result.add("data", replies); + snowflakereturnstring = result.toString(); + + } catch (Exception e) { + + System.out.println("in exception with " + e.getMessage()); + snowflakereturnstring = "exception "; + if (returnciphertextbool) { + if (e.getMessage().contains("1401") + || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) { + + result = new JsonObject(); + replies = new JsonArray(); + rownbranddata = new JsonArray(); + + for (int i = 0; i < snowflakedata.size(); i++) { + + JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray(); + + for (int j = 0; j < snowflakerow.size(); j++) { + if (j == 1) { + // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString(); + // FPE example + String sensitive = checkValid(snowflakerow); + + if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) { + if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) { + if (sensitive.contains("notvalid")) { + System.out.println("adding null not charint or nbr"); + sensitive = sensitive.replace("notvalid", ""); + } else + sensitive = BADDATATAG; + + } else if (sensitive.equalsIgnoreCase("null") + || sensitive.equalsIgnoreCase("notvalid")) { + + } else if (sensitive.contains("notvalid")) { + sensitive = sensitive.replace("notvalid", ""); + + } + encdata = sensitive; + + } else { + System.out.println("normal number data" + sensitive); + } + rownbranddata.add(sensitive); + replies.add(rownbranddata); + rownbranddata = new JsonArray(); + + } else { + JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive(); + int row_number = snowflakecolumn.getAsInt(); + rownbranddata.add(row_number); + } + } + } + + result.add("data", replies); + JsonObject inputJsonObject = new JsonObject(); + String bodyString = result.toString(); + inputJsonObject.addProperty("statusCode", 200); + inputJsonObject.addProperty("body", bodyString); + + snowflakereturnstring = inputJsonObject.toString(); + System.out.println(" new data " + snowflakereturnstring); + + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + } else { + statusCode = 400; + snowflakereturnstring = formatReturnValue(statusCode); + e.printStackTrace(System.out); + } + + } finally { + + } + System.out.println(snowflakereturnstring); + // response.getWriter().write(snowflakereturnstring); + + } + + public String formatReturnValue(int statusCode) + + { + StringBuffer snowflakereturndatasb = new StringBuffer(); + + snowflakereturndatasb.append("{ \"statusCode\":"); + snowflakereturndatasb.append(statusCode); + snowflakereturndatasb.append(","); + snowflakereturndatasb.append(" \"body\": {"); + snowflakereturndatasb.append(" \"data\": ["); + snowflakereturndatasb.append("] }}"); + System.out.println("in exception with "); + return snowflakereturndatasb.toString(); + } + + public String checkValid(JsonArray snowrow) { + String inputdata = null; + String notvalid = "notvalid"; + if (snowrow != null && snowrow.size() > 0) { + JsonElement element = snowrow.get(1); + if (element != null && !element.isJsonNull()) { + inputdata = element.getAsString(); + if (inputdata.isEmpty() || inputdata.length() < 2) { + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("Sensitive data is null or empty."); + inputdata = notvalid + inputdata; + } + } else { + // System.out.println("bigquerytrow is null or empty."); + inputdata = notvalid + inputdata; + } + + return inputdata; + + } + + public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID, + String userSetLookupIP) throws Exception { + + CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP); + + String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd); + String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken); + + boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken); + + return founduserinuserset; + + } + + @Override + public void service(HttpRequest request, HttpResponse response) throws Exception { + // TODO Auto-generated method stub + + } +} \ No newline at end of file