type) throws SQLException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ // Add other methods as needed
+}
diff --git a/database/postgress/src/main/java/com/ThalesEncryptDecryptService.java b/database/postgress/src/main/java/com/ThalesEncryptDecryptService.java
new file mode 100644
index 00000000..33c40f4d
--- /dev/null
+++ b/database/postgress/src/main/java/com/ThalesEncryptDecryptService.java
@@ -0,0 +1,156 @@
+import java.io.InputStream;
+import java.security.Provider;
+import java.security.Security;
+import java.util.Properties;
+
+import javax.crypto.Cipher;
+
+import com.ingrian.security.nae.FPECharset;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec;
+import com.ingrian.security.nae.IngrianProvider;
+import com.ingrian.security.nae.NAEKey;
+import com.ingrian.security.nae.NAESession;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
+import com.ingrian.security.nae.IngrianProvider.Builder;
+
+
+public class ThalesEncryptDecryptService {
+
+ private static final IngrianProvider provider;
+ private static Properties properties;
+
+ static {
+ try (InputStream input = ThalesEncryptDecryptService.class.getClassLoader()
+ .getResourceAsStream("application.properties")) {
+ properties = new Properties();
+ if (input == null) {
+ throw new RuntimeException("Unable to find application.properties");
+ }
+ properties.load(input);
+ } catch (Exception ex) {
+ throw new RuntimeException("Error loading properties file", ex);
+ }
+ }
+
+ static {
+ try {
+ // Load the properties file as an InputStream
+ System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
+ "CADP_for_JAVA.properties");
+ // InputStream inputStream =
+ // ThalesDataBricksCADPFPE.class.getClassLoader().getResourceAsStream("CADP_for_JAVA.properties");
+ InputStream inputStream = ThalesEncryptDecryptService.class.getClassLoader()
+ .getResourceAsStream("CADP_for_JAVA.properties");
+ if (inputStream == null) {
+ throw new RuntimeException("Failed to find CADP_for_JAVA.properties file.");
+ }
+
+ // Initialize the IngrianProvider using the static context
+ provider = new IngrianProvider.Builder().addConfigFileInputStream(inputStream).build();
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new RuntimeException("Failed to initialize IngrianProvider.", e);
+ }
+ }
+
+
+
+ public static String callDecryptApi(String encryptedData) throws Exception {
+
+ //System.out.println("Data to decrypt \"" + encryptedData + "\"");
+ NAESession session = null;
+ String username = properties.getProperty("app.cmuser");
+ String password = properties.getProperty("app.cmpwd");
+ String keyName = properties.getProperty("app.keyname");
+
+ String tweakAlgo = null;
+ String tweakData = null;
+ String encdata = encryptedData;
+
+ try {
+ // create NAE Session: pass in NAE user name and password
+ session = NAESession.getSession(username, password.toCharArray());
+
+ // Get SecretKey (just a handle to it, key data does not leave the server
+ NAEKey key = NAEKey.getSecretKey(keyName, session);
+ String algorithm = "FPE/FF1/CARD62";
+ FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
+ .build();
+ // get a cipher
+ Cipher thalesCipher = Cipher.getInstance(algorithm, "IngrianProvider");
+ // initialize cipher to decrypt.
+ thalesCipher.init(Cipher.DECRYPT_MODE, key, param);
+
+ // decrypt data
+ byte[] outbuf = thalesCipher.doFinal(encryptedData.getBytes());
+
+ encdata = new String(outbuf);
+
+ // close the session
+ session.closeSession();
+ } catch (Exception e) {
+ System.out.println("The Cause is " + e.getMessage() + ".");
+ if (!e.getMessage().startsWith("User is not authorized to perform this operation at this time for") && !e.getMessage().contains("1401"))
+ throw e;
+
+
+ } finally {
+ if (session != null) {
+ session.closeSession();
+ }
+ }
+ return new StringBuilder(encdata).toString(); // Mock decryption (reversing string)
+ // return new StringBuilder(encdata).reverse().toString(); // Mock decryption (reversing string)
+ }
+
+ public static String callEncrypptApi(String sensitive) throws Exception
+ {
+
+
+ //System.out.println("Data to encrypt \"" + sensitive + "\"");
+ NAESession session = null;
+ String username = properties.getProperty("app.cmuser");
+ String password = properties.getProperty("app.cmpwd");
+ String keyName = properties.getProperty("app.keyname");
+
+ String tweakAlgo = null;
+ String tweakData = null;
+ String encdata = null;
+
+ try {
+ // create NAE Session: pass in NAE user name and password
+ session = NAESession.getSession(username, password.toCharArray());
+
+ // Get SecretKey (just a handle to it, key data does not leave the server
+ NAEKey key = NAEKey.getSecretKey(keyName, session);
+ String algorithm = "FPE/FF1/CARD62";
+ FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
+ .build();
+ // get a cipher
+ Cipher encryptCipher = Cipher.getInstance(algorithm, "IngrianProvider");
+ // initialize cipher to encrypt.
+ encryptCipher.init(Cipher.ENCRYPT_MODE, key, param);
+
+ // encrypt data
+ byte[] outbuf = encryptCipher.doFinal(sensitive.getBytes());
+
+ encdata = new String(outbuf);
+
+ // close the session
+ session.closeSession();
+ } catch (Exception e) {
+ System.out.println("The Cause is " + e.getMessage() + ".");
+ if (!e.getMessage().startsWith("User is not authorized to perform this operation at this time for") && !e.getMessage().contains("1401"))
+ throw e;
+
+ } finally {
+ if (session != null) {
+ session.closeSession();
+ }
+ }
+
+ return encdata;
+
+ }
+
+}
diff --git a/database/postgress/src/main/java/com/ThalesExampleWithWrapper.java b/database/postgress/src/main/java/com/ThalesExampleWithWrapper.java
new file mode 100644
index 00000000..8c217881
--- /dev/null
+++ b/database/postgress/src/main/java/com/ThalesExampleWithWrapper.java
@@ -0,0 +1,54 @@
+import java.io.InputStream;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Properties;
+
+public class ThalesExampleWithWrapper {
+
+
+ private static Properties properties;
+
+ static {
+ try (InputStream input = ThalesExampleWithWrapper.class.getClassLoader()
+ .getResourceAsStream("application.properties")) {
+ properties = new Properties();
+ if (input == null) {
+ throw new RuntimeException("Unable to find application.properties");
+ }
+ properties.load(input);
+ } catch (Exception ex) {
+ throw new RuntimeException("Error loading properties file", ex);
+ }
+ }
+
+ public static void main(String[] args) {
+ String url = properties.getProperty("db.url");
+ String user = properties.getProperty("db.username");
+ String password = properties.getProperty("db.password");
+
+ try (
+
+ Connection conn = DriverManager.getConnection(url, user, password);
+ Statement stmt = conn.createStatement()) {
+
+
+ String query = "SELECT custid, email, address FROM plaintext_protected limit 5";
+ try (ResultSet rs = new ThalesDecryptingResultSetWrapper(stmt.executeQuery(query))) {
+ while (rs.next()) {
+ String id = rs.getString("custid");
+ // int id = rs.getInt("custid");
+ String email = rs.getString("email");
+ String address = rs.getString("address");
+ System.out.println("ID: " + id);
+ System.out.println("Email: " + email);
+ System.out.println("Address: " + address);
+
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+}
diff --git a/database/postgress/src/main/resources/CADP_for_JAVA.properties b/database/postgress/src/main/resources/CADP_for_JAVA.properties
new file mode 100644
index 00000000..5684dbb4
--- /dev/null
+++ b/database/postgress/src/main/resources/CADP_for_JAVA.properties
@@ -0,0 +1,657 @@
+#
+# Copyright (c) 2002-2020 SafeNet, Inc.
+#
+#
+# Ingrian Network-Attached Encryption (NAE) properties file
+#
+# Release Version: 8.12.0.000-000
+#
+
+#[Version]
+# Version of the properties file for the Ingrian JCE provider.
+#
+# Do not modify this property.
+#
+Version=2.4
+
+
+#[Network Configuration]
+# [NAE Server IP and Port]
+# The IP address and port of the NAE server.
+#
+# Multiple IPs can be specified when load balancing is used.
+# For all tier-aware parameters, the tier is indicated with a trailing
+# :n after the parameter name, i.e. NAE_IP.1=127.0.0.1
+# Setting the parameter with no tier sets the default value for all tiers.
+# i.e. Connection_Timeout=600000 sets Connection_Timeout for all tiers while
+# Connection_Timeout.1=700000 sets Connection_Timeout for tier 1.
+#
+# Multiple IPs are separated by colons, e.g.,
+# 192.168.1.10:192.168.1.11:192.168.1.12
+# For IPv6, the IP address is to be specified in curly braces, such as {2001:0db8:85a3:0000:0000:8a2e:0370:7334}. Also, combination of IPv4 and IPv6 IP
+# addresses can be used separated by colons(:) provided each IPv6 IP address is within {}. The IPv6 is supported for the
+# KMIP interfaces too.
+# {IPv6}:IPv4:{IPv6}
+# IPv4:{IPv6}
+# IPv4:{IPv6}:IPv4
+# {IPv6}:IPv4:IPv4:{IPv6}
+#NAE_IP.1=52.86.120.81
+NAE_IP.1=yourip
+
+#[Network Configuration]
+# [NAE Server Port]
+# NAE_Port is tier-aware
+# Do not set the port value to 9443 because this is the port typically used
+# to connect to the management console.
+NAE_Port=9001
+
+#[Network Configuration]
+# [Ignore_DNS_Resolution_Failure]
+# Ignore_DNS_Resolution_Failure is used in case either JCE Application fails to locate DNS Server.
+# or If Host Name present in any Ip Field is not getting resolved through DNS.Application will shift to Persistance chache if it is on.
+Ignore_DNS_Resolution_Failure=false
+
+#[Network Configuration]
+# [KMIP Server Port]
+# KMIP_Port is tier-aware
+# By convention, the KMIP standard port is 5696, but this is not enforced,
+# or taken as default by JCE. Must be a KMIP SSL port on the server.
+# Do not set the port value to 9443 because this is the port typically used
+# to connect to the management console.
+KMIP_Port=5696
+
+#[Network Configuration]
+# [Protocol]
+# The protocol used between the client and the NAE server.
+#
+# Protocol is tier-aware.
+# Valid values: tcp, ssl.
+# Default: tcp
+# Recommended: ssl
+#
+Protocol=tcp
+
+# All connection configuration properties effect the connection set up.
+#
+#[Connection Configuration]
+# [Verify_SSL_Certificate]
+# This property is considered only when Protocol is set to ssl.
+#
+# Enable or disable verification of DataSecure IP address/host name
+# against Subject Common Name (CN) or Subject Alternative Name (DNS or IP)
+# in the certificate.
+#
+# Valid values: yes, no.
+# Default: no
+# Recommended: yes
+#
+Verify_SSL_Certificate=no
+
+#[Connection Configuration]
+# [SSL_Handshake_Timeout]
+# This property is considered only when Protocol is set to ssl.
+#
+# Allocates a time for SSL handshake. If SSL handshake is not complete
+# within this time period, the connection is closed.
+#
+# The value is specified in milliseconds.
+#
+# If the value is set to 0 or is not set, SSL handshake timeout
+# is not enforced.
+#
+SSL_Handshake_Timeout=
+
+#[Connection Configuration]
+# [Persistent Connections]
+# Enable or disable persistent connections.
+#
+# If enabled, the client will use a pool of persistent connections to the
+# NAE server. If disabled, a new connection will be created and then
+# closed for each request.
+#
+# Valid values: yes, no.
+# Default: yes
+# Recommended: yes
+#
+Use_Persistent_Connections=yes
+
+
+#[Connection Configuration]
+# [Connection Pooling]
+# The maximum number of connections in the persistent connection pool per session.
+#
+# This value is used only when persistent connections are enabled.
+#
+# Size_of_Connection_Pool is tier-aware.
+# Default: 300
+#
+Size_of_Connection_Pool=300
+
+#[Connection Configuration]
+#[Load Balancing Configuration]
+# The type of load balancing.
+#
+# This value is only relevant if you are load balancing across multiple
+# NAE servers.
+#
+# Valid values: random, round-robin.
+# Default: round-robin
+#
+Load_Balancing_Algorithm=round-robin
+
+#[Connection Configuration]
+# [Connection Idle Timeout]
+# The time a connection is allowed to be idle in the connection pool
+# before it gets closed automatically by the client.
+#
+# The timeout can be specified in any time units (default - milliseconds). The client
+# will check how long each connection has been idle for. If the time has passed the value
+# specified here, the client will close the connection and remove it from
+# the connection pool. To be effective, this setting must be less than the
+# Connection Timeout setting in the NAE Server Settings section in the
+# Management Console of the NAE server.
+#
+# Setting this value to 0 is equivalent to an infinite timeout.
+# Connection_Idle_Timeout is tier-aware.
+# Default: 600000
+#
+Connection_Idle_Timeout=600000
+
+#[Connection Configuration]
+# [Unreachable Server Retry]
+# The amount of time to try establishing a connection on a load balancer with the server.
+#
+# The retry period can be specified in any time units (default - milliseconds). An error is returned
+# after the specified period if no server in the pool becomes reachable.
+# If logging is enabled, error messages will be logged to the log file.
+#
+# Setting this value to -1 is equivalent to an infinite retry period. The
+# client will keep trying to connect to a server until a connection is
+# established.
+#
+# Setting this value to -1 is not compatible with multi-tier load
+# balancing because the load balancer will never switch to the secondary
+# or tertiary pools.
+#
+# Setting the value 0 means no retrying on the particular load balancer in case all the server
+# are down. It will move to next load balancer if available.
+#
+# Default: 60000 (1 minute)
+#
+Unreachable_Server_Retry_Period=60000
+
+#[Connection Configuration]
+# [Maximum_Server_Retry_Period]
+# The total amount of time to spend trying to make connections on any tier.
+# This value only has meaning when using multi-tiered load balancing.
+# If this value is set to -1 (try forever), the connection manager will try
+# every server on every tier continually, until one answers.
+# If this value is enabled, the connection manager will try to make connections
+# for at least Maximum_Server_Retry_Period time but will return
+# an error if no connection can be made on the tier in use when
+# Maximum_Server_Retry_Period expires.
+# Server tries to make connection on each tier serially. So, new connection will
+# be created whether other connection mark any tier disable or not.
+# Maximum_Server_Retry_Period is tier-aware.
+#
+# Default: 0 (disabled)
+#
+Maximum_Server_Retry_Period=0
+
+#[Connection Configuration]
+# [Connection Timeout]
+# The timeout when connecting to the NAE server.
+#
+# The timeout can be specified in any time units (default - milliseconds). The client
+# will wait for the specified time when trying to connect to each NAE
+# server. If the timeout expires, an NAEException will be thrown.
+#
+# Setting this value to 0 is equivalent to an infinite timeout.
+#
+# Caution: Setting this value too low (but not 0) may cause connections to
+# fail when the NAE servers and/or network are under load.
+#
+# Connection_Timeout is tier-aware.
+# Default: 1 minute
+#
+Connection_Timeout=1m
+
+
+#[Connection Configuration]
+# [Connection Read Timeout]
+# The timeout when reading from the NAE server.
+#
+# The timeout can be specified in any time units (default - milliseconds). The client
+# will wait for the specified time when trying to read data from the NAE
+# server. If the timeout expires, an NAEException will be thrown.
+#
+# Setting this value to 0 is equivalent to an infinite timeout.
+# Connection_Read_Timeout is tier-aware.
+# Default: 7000
+#
+Connection_Read_Timeout=7000
+
+#[Connection Configuration]
+# [Connection Retry]
+# The amount of time to wait before trying to reconnect to a disabled
+# server.
+#
+# The retry interval can be specified in any time units (default - milliseconds).
+# If one of the NAE servers in a load balanced configuration is not reachable,
+# the client will disable this server, and then wait for the specified
+# time before trying to connect to it again.
+#
+# Setting this value to 0 is equivalent to an infinite retry interval
+# (meaning the disabled server will never be brought back into use).
+#
+# Connection_Retry_Interval is tier-aware.
+#
+# Default: 600000
+#
+Connection_Retry_Interval=600000
+
+#[Client Certificate Configuration]
+# [Client Certificate Alias]
+# The client certificate to present to the NAE server.
+#
+# This value is only relevant when client certificate authentication is
+# enabled on the NAE server. The certificate must be in PEM format.
+#
+# When there are multiple client certificates in your keystore, you can
+# specify which certificate gets sent to the NAE server. If you do not
+# specify an alias, the first certificate in the keystore will be sent.
+#
+# You should provide the alias of the client certificate.
+# Client_Cert_Alias is tier-aware.
+# No default.
+#
+Client_Cert_Alias=
+
+
+#[Client Certificate Configuration]
+# [Client Private Key Passphrase]
+# The passphrase to unlock the client private key.
+# Client_Cert_Passphrase is tier-aware.
+# No default.
+#
+Client_Cert_Passphrase=
+#[Etoken Configuration]
+# Enable or disable Etoken.
+#
+# If enabled, the client will use a Etoken to make ssl connection with
+# DS server. If disabled, a new connection will be created using local
+# keystore for SSL Connection. This configuration will work only when
+# protocol is ssl.
+#
+# Valid values: yes, no.
+# Default: no
+Use_Etoken=no
+#[Etoken Configuration]
+# The Etoken Name is used to connect with
+# PKCS#11 hardware.
+#
+Etoken_Name=
+
+#[SSL/TLS Configuration]
+#[Certificate Configuration]
+# Location of the key store where user stores certificates (both CA certificates and end user certificates)
+# and private keys.
+# Key_Store_Location is tier-aware.
+#
+# Or Location of Etoken library in case on Use_Etoken is enabled. Etoken Library is required to load
+# Etoken Keystore.
+# No default. ;
+Key_Store_Location=C:\\code\\aws-lambda\\certs\\cacerts
+
+#[SSL/TLS Configuration]
+#[Certificate Configuration]
+# Password to unlock keystore
+# Key_Store_Password is tier-aware.
+# Or
+# Etoken Pin is case of Use_Etoken is enabled.
+# No default.
+Key_Store_Password=yorupwd
+
+#[Connection Configuration]
+# [Cluster_Synchronization_Delay]
+# The total amount of time to spend trying to make requests on keys
+# go to the same device the key create or latest key modify went to.
+#
+# A device tries to replicate key information to other devices in the
+# cluster after it completes a key create or modify request. Until
+# that replication completes, requests on the key need to go to the
+# device pushing the replication.
+#
+# If replication fails, the device waits for 40 seconds, then
+# tries again. If four replications fail, the device stops trying
+# to replicate data.
+#
+# The default is 170 seconds: 4 times 40 seconds plus a few extra
+# seconds per try for network latency. For larger clusters additional
+# time may be needed.
+#
+# Disable the function: 0 seconds
+#
+# Default: 170 seconds
+#
+Cluster_Synchronization_Delay=170
+
+# [Client Key Caching]
+# [Symmetric_Key_Cache_Enabled]
+# Enables symmetric key caching.
+#
+# If enabled, the client will be able to use symmetric keys to encrypt data
+# locally. If disabled, only remote encryption will be supported. Should only be
+# enabled with Protocol set to ssl. To allow key caching over unsecured
+# communication, set this variable to tcp_ok
+#
+# Valid values: yes, no, tcp_ok
+# Default: no
+# Recommended: no
+Symmetric_Key_Cache_Enabled=tcp_ok
+Asymmetric_Key_Cache_Enabled=tcp_ok
+
+# [Client Key Caching]
+# [Symmetric_Key_Cache_Expiry]
+# Time period since key was cached after which a symmetric key
+# may be removed from cache. Symmetric_Key_Cache_Expiry can be specified
+# in any time units (default - seconds)
+# Setting this value to 0 is equivalent to an infinite timeout.
+# Note: This field is also applicable to Asymmetric key cache expiry
+# Default: 43200 (12 hours)
+Symmetric_Key_Cache_Expiry=43200
+
+# [Client Key Caching]
+# [Symmetric_Key_Cache_AutoRefresh_Interval]
+# It is a time duration after which a cached key will become eligible for refresh in cache.
+# Actual key refresh will only be initiated when the cached key is queried from cache after
+# it becomes eligible for refresh before its expiry time. If Persistent Cache is enabled then Key
+# will be refreshed both in Symmetric and Persistent Cache during refresh operation.
+# Auto refresh cache operation will only be enabled when this interval value is greater than 0.
+# Symmetric_Key_Cache_AutoRefresh_Interval can be specified in any time unit (default unit -seconds).
+#default :0
+
+Symmetric_Key_Cache_AutoRefresh_Interval=0
+
+
+# [Client Key Caching]
+# [Local_Cipher_Cache_Expiry]
+# Time period since local cipher was initialize with cached key.
+# Local_Cipher_Expiry can be specified in any time units (default - milliseconds)
+# Setting this value to 0 is equivalent to no timeout which means cipher will
+# expire after every operation.
+# To do local cipher expiry, Symmetric_Key_Cache_Enabled must be set to
+# "yes" or "tcp_ok.
+#
+# Note: Local_Cipher_Expiry timeout should be less than Symmetric_Key_Cache_Expiry
+# because all ciphers will get expired on symmetric cache expiry.
+# Default: 0 (cipher will expiry every time)
+Local_Cipher_Cache_Expiry=0
+
+# [Client Key Caching]
+# [Local_Crypto_Provider]
+# Name of JCE provider to perform local crypto
+#
+# Default: SunJCE or IBMJCE (depends on JVM)
+Local_Crypto_Provider=
+
+# [Persistent Key Caching]
+# [Persistent_Cache_Enabled]
+# Enables persistent key caching during local encryption.
+#
+# To persist symmetric keys Symmetric_Key_Cache_Enabled must be set to
+# "yes" or "tcp_ok", Persistent_Cache_Enabled must be set to "yes",
+# Persistent_Cache_Expiry set to a zero or positive value, and
+# Persistent_Cache_Directory set to directory.
+# If Symmetric_Key_Cache_Enabled or Public_Key_Cache_Enabled is set
+# to "no", all Persistent_Cache_* properties will be ignored.
+#
+# Valid values: yes, no
+# Default: no
+# Recommended: no
+Persistent_Cache_Enabled=no
+
+
+# [Persistent Key Caching]
+# [Persistent_Cache_Expiry_Keys]
+#
+# Time since stored after which a key may be removed from key cache.
+# Settiing to 0 would mean no timeout and keys would not be removed
+# from key cache. Persistent_Cache_Expiry_Keys can be specified
+# in any time units (default - seconds)
+#
+# Default: 43200 (12 hours)
+Persistent_Cache_Expiry_Keys=43200
+
+# [Persistent Key Caching]
+# [Persistent_Cache_Directory]
+# Specifies the location of the persistent cache; error if not specified.
+#
+# Default: none
+#
+Persistent_Cache_Directory=
+
+# [Persistent Key Caching]
+# [Persistent_Cache_Max_Size]
+# Limits the number of keys, certificates, and crypto profiles in persistent cache
+#
+# Default: 100
+Persistent_Cache_Max_Size=100
+
+
+#########################################################################
+# THE FOLLOWING PROPERTIES ONLY NEED TO BE SET IF YOU ARE INSTALLING THE
+# INGRIAN JCE PROVIDER AS PART OF AN ORACLE OR DB2 DATABASE CONNECTOR;
+# OTHERWISE, THEY ARE DISREGARDED.
+#########################################################################
+
+#[SSL/TLS Configuration]
+# [CA Certificate for Server Authentication]
+# The CA certificate that signed the NAE server certificate presented to
+# clients to establish SSL connections.
+#
+# If you are using SSL between the client and server, you must specify a
+# path to the CA certificate that signed the NAE server certificate. If
+# the client cannot validate the certificate presented by the NAE server,
+# the client will not be able to establish an SSL connection with the NAE
+# server.
+#
+# You should provide the path and file name of the CA certificate. The
+# path can be absolute or relative to the application. Do not use quotes
+# when specifying the path, even if it contains spaces.
+#
+# No default.
+#
+CA_File=
+
+
+#[SSL/TLS Configuration]
+# [Client Certificate]
+# The client certificate to present to the NAE server.
+#
+# This value is required when client certificate authentication is enabled
+# on the NAE server. The certificate must be in PEM format. If this value
+# is set, the certificate and private key must be present even if the NAE
+# server is not configured to request a client certificate.
+#
+# You should provide the path and file name of the client certificate. The
+# path can be absolute or relative to the application. Do not use quotes
+# when specifying the path, even if it contains spaces.
+#
+# No default.
+#
+Cert_File=
+
+
+#[SSL/TLS Configuration]
+# [Client Private Key]
+# The private key associated with the client certificate specified in
+# Cert_File.
+#
+# This value is required when client certificate authentication is enabled
+# on the NAE server. The client private key must be in PEM-encoded PKCS#12
+# format. If this value is set, a correctly formatted key and certificate
+# must be present.
+#
+# You should provide the path and file name of the private key. The path
+# can be absolute or relative to the application. Do not use quotes when
+# specifying the path, even if it contains spaces.
+#
+# No default.
+#
+Key_File=
+
+
+#[SSL/TLS Configuration]
+# [Client Private Key Passphrase]
+# The passphrase to unlock the client private key specified in Key_File.
+#
+# This value is required when client certificate authentication is enabled
+# on the NAE server. Since the value is in the clear, this properties file
+# must have its permission restricted so that it can be read only by the
+# applications that are to have legitimate access to it.
+#
+# No default.
+#
+Passphrase=
+
+#[This flag indicates that NAE USERNAME and PASSWORD are encrypted using Passpharse utility]
+#default: no
+Credentials_Encrypted=no
+
+#[This flag indicates that client certificate passphrase and keystore password is encrypted using Passpharse utility]
+#default: no
+Passphrase_Encrypted=no
+
+#[Logging Configuration]
+# [Log Level]
+# The level of logging that will be performed by the client.
+#
+# The log level determines how verbose your client logs are. You can
+# disable logging by selecting NONE; however, it is recommended that you
+# set the log level to LOW. A log level of HIGH can create a very large
+# log file. Set the log level to HIGH to troubleshoot configuration
+# problems.
+#
+# Valid values:
+# NONE No logging
+# LOW Errors only
+# MEDIUM Warnings
+# HIGH Info
+# DEBUG Debug
+# Default: LOW
+#
+Log_Level=NONE
+
+
+#[Logging Configuration]
+# [Log File]
+# The location of the log file the client will create.
+#
+# You should provide the path and file name of the log file. The path can
+# be absolute or relative to the application. Do not use quotes when
+# specifying the path, even if it contains spaces.
+#
+# No default.
+#
+Log_File=
+
+
+#[Logging Configuration]
+# [Log Rotation]
+# The log rotation method.
+#
+# This value specifies how frequently the log file is rotated.
+#
+# Valid values:
+# none - log file is not rotated
+# Daily - log file is rotated once a day
+# Weekly - log file is rotated once a week
+# Monthly - log file is rotated once a month
+# Size - log file is rotated when it exceeds Log_Size_Limit
+#
+# Default: Daily
+#
+Log_Rotation=Daily
+
+#[Logging Configuration]
+# [Log time representation]
+#
+# This value specifies if timestamp should be formatted in GMT or not.
+#
+# Valid values:
+# yes - timestamps will be formatted in GMT
+# no - timestamps will not be formatted in GMT
+#
+# Default: no
+#
+Log_GMT=no
+
+
+#[Logging Configuration]
+# [Log Size]
+# The maximum log file size.
+#
+# If Log_Rotation=Size, the log will be rotated after it reaches the
+# specified size. This value is only used when Log_Rotation=Size.
+#
+# The size may be specified in bytes, kilobytes (using 'k' or 'K'), or
+# megabytes (using 'm' or 'M'). One kilobyte is 1024 bytes, and one
+# megabyte is 1048576 bytes.
+#
+# Default: 100k
+#
+Log_Size_Limit=100k
+
+#[SYSLOG IP]
+# The IP address of SYSLOG server.
+# No Default
+SysLog_IP=
+
+#[SYSLOG Port]
+# The port of SYSLOG server.
+# No Default
+SysLog_Port=
+
+#[SYSLOG Protocol]
+# The Protocol of SYSLOG server Either TCP or UDP or SSL.
+# In case of SSL, add CA certificate and private key into keyStore
+# No Default
+SysLog_Protocol=
+
+#[SYSLOG SSL KeyStore Location]
+# Location of the keystore/truststore having SYSLOG server certificates and CA.
+# It is required only in case if [SYSLOG Protocol] is SSL
+# By Default we will cacert from jre_home/lib/security will be referred from
+SysLog_SSLKeystore=
+
+#[SYSLOG SSL KeyStore Password]
+# Password of the keystore/truststore having SYSLOG server certificates and CA.
+# It is required only in case if [SYSLOG Protocol] is SSL
+# By Default cacert's password is null for java application.
+SysLog_SSLKeystorePassword=
+
+#[Logging Configuration]
+# [Additional log4j properties]
+# File containing additional log4j configuration properties.
+#
+# Full path to the log4j configuration file, where user can specify additional
+# configuration properties in "key=value" format
+# For example, to set the maximum number of backup files to keep around,
+# user can set "log4jIngrian.appender.ingrian.logfileAppender.MaxBackupIndex=10"
+#
+# Default: none
+Log_Config_Advanced=
+
+#[Key_non_exportable_policy]
+# For non exportable key in local caching mode yes means it does cipher operation remotely
+# and for no means it will not go for remote operation if key is non exportable
+# default = no
+Key_non_exportable_policy=no
+
+#[Log_MaxBackupIndex]
+# Maximum number of log backup files to be stored on the disk,if log rotation is enabled
+# Supports integer values. If 0 or less than -1 values are supplied then maximum 7 backup files will be created
+#
+# default = -1, which means infinite number of backup files
+Log_MaxBackupIndex=-1
diff --git a/database/postgress/src/main/resources/application.properties b/database/postgress/src/main/resources/application.properties
new file mode 100644
index 00000000..6735cfa6
--- /dev/null
+++ b/database/postgress/src/main/resources/application.properties
@@ -0,0 +1,22 @@
+# Database connection settings
+db.url=jdbc:postgresql://yourpostgressdbip:5432/mydatabase
+db.username=posgressuser
+db.password=posgressuserpwd!
+
+# Connection pool settings (optional)
+db.initialSize=5
+db.maxActive=10
+db.maxIdle=5
+db.minIdle=2
+
+# Application-specific settings
+app.keyname=yourcmkey
+app.cmuser=appteam1
+app.cmpwd=Yourcmuserpwd!
+app.fieldstoprotect=email,address
+
+# Logging settings
+logging.level.root=INFO
+logging.level.com.example=DEBUG
+
+
diff --git a/database/redshift/documentation/AWS-Redshift_with_CADP.pdf b/database/redshift/documentation/AWS-Redshift_with_CADP.pdf
new file mode 100644
index 00000000..f503fc98
Binary files /dev/null and b/database/redshift/documentation/AWS-Redshift_with_CADP.pdf differ
diff --git a/database/redshift/documentation/AWS-Redshift_with_CRDP.pdf b/database/redshift/documentation/AWS-Redshift_with_CRDP.pdf
new file mode 100644
index 00000000..ae3a7ed7
Binary files /dev/null and b/database/redshift/documentation/AWS-Redshift_with_CRDP.pdf differ
diff --git a/database/redshift/pom.xml b/database/redshift/pom.xml
new file mode 100644
index 00000000..3a935293
--- /dev/null
+++ b/database/redshift/pom.xml
@@ -0,0 +1,145 @@
+
+
+
+ 4.0.0
+ Thales
+ CADP-AWS-Redshift-UDF
+ 0.0.2-SNAPSHOT
+
+
+ 11
+ 11
+ 3.12.0
+ 4.4
+ 1.70
+ 1.10
+
+ 31.1-jre
+ 2.9.0
+ 2.17.2
+ 2.17.2
+ .000
+
+
+
+
+ org.apache.commons
+ commons-lang3
+ 3.12.0
+
+
+ io.github.thalescpl-io.cadp
+ CADP_for_JAVA
+ 8.15.0.001
+
+
+ javax.xml.bind
+ jaxb-api
+ 2.3.1
+
+
+
+ commons-codec
+ commons-codec
+ ${org.apache.commons.codec.version}
+
+
+ com.google.guava
+ guava
+ ${guava.version}
+
+
+ com.google.code.findbugs
+ jsr305
+
+
+ com.google.errorprone
+ error_prone_annotations
+
+
+ com.google.guava
+ listenablefuture
+
+
+ com.google.j2objc
+ j2objc-annotations
+
+
+ org.checkerframework
+ checker-qual
+
+
+
+
+ com.google.code.gson
+ gson
+ ${gson.version}
+
+
+ org.apache.commons
+ commons-collections4
+ 4.4
+
+
+ com.squareup.okhttp3
+ okhttp
+ 4.10.0
+
+
+ commons-io
+ commons-io
+ 2.4
+
+
+ com.amazonaws
+ aws-lambda-java-core
+ 1.2.1
+
+
+ com.amazonaws
+ aws-lambda-java-events
+ 3.11.0
+
+
+ com.amazonaws
+ aws-lambda-java-log4j2
+ 1.5.1
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+ 3.2.2
+
+ false
+
+
+
+ package
+
+ shade
+
+
+
+
+
+
+
+
+
+
+
+ com.github.edwgiz
+ maven-shade-plugin.log4j2-cachefile-transformer
+ 2.13.0
+
+
+
+
+
+ CADP-AWS-Redshift-UDF
+
diff --git a/database/redshift/src/main/java/example/CMUserSetHelper.java b/database/redshift/src/main/java/example/CMUserSetHelper.java
index 6621de5d..06f17015 100644
--- a/database/redshift/src/main/java/example/CMUserSetHelper.java
+++ b/database/redshift/src/main/java/example/CMUserSetHelper.java
@@ -1,5 +1,4 @@
-package example;
-
+package com.example;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
@@ -57,11 +56,10 @@ public class CMUserSetHelper {
static int CHUNKSIZEMAX = 100;
public CMUserSetHelper(String usersetid, String cmIP) {
-
+
this.usersetid = usersetid;
this.cmIP = cmIP;
- this.apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?name=";
- //this.apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?limit=" + this.usersetlimit + "&name=";
+ this.apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?name=";
this.addusertouserset = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users";
this.authUrl = "https://" + cmIP + "/api/v1/auth/tokens";
}
@@ -73,19 +71,15 @@ public static void main(String[] args) throws Exception {
String cmip = args[2];
String usersetid = args[3];
String filePath = args[4];
- // CMUserSetHelper("716f01a6-5cab-4799-925a-6dc2d8712fc1","20.241.70.238");
- //CMUserSetHelper("32d89a8d-efac-4c50-9b53-f51d0c03413e",
- CMUserSetHelper cmusersetHelper = new CMUserSetHelper(usersetid,
- cmip);
+ CMUserSetHelper cmusersetHelper = new CMUserSetHelper(usersetid, cmip);
int totalrecords = 0;
- // String apiUrl = apiUrlGetUsers;
- String jwthtoken = geAuthToken(cmusersetHelper.authUrl,username, password);
+ String jwthtoken = geAuthToken(cmusersetHelper.authUrl, username, password);
String newtoken = "Bearer " + removeQuotes(jwthtoken);
-
- // String filePath = "C:\\Users\\t0185905\\workspace\\CT-VL-GCP\\src\\main\\java\\com\\example\\emailAddresses.txt";
+ // String filePath =
+ // "C:\\Users\\t0185905\\workspace\\CT-VL-GCP\\src\\main\\java\\com\\example\\emailAddresses.txt";
RandomAccessFile file = new RandomAccessFile(filePath, "r");
if (cmusersetHelper.chunksize > CHUNKSIZEMAX)
cmusersetHelper.chunksize = CHUNKSIZEMAX;
@@ -93,6 +87,7 @@ public static void main(String[] args) throws Exception {
if (cmusersetHelper.chunksize > totoalnbrofrecords) {
cmusersetHelper.chunksize = totoalnbrofrecords / 2;
}
+ //totalrecords = cmusersetHelper.addAUserToUserSet(cmusersetHelper.addusertouserset, newtoken);
totalrecords = cmusersetHelper.addAUserToUserSetFromFile(cmusersetHelper.addusertouserset, newtoken, filePath);
System.out.println("Totalrecords inserted into Userset " + cmusersetHelper.usersetid + " = " + totalrecords);
@@ -102,12 +97,10 @@ public static void main(String[] args) throws Exception {
* Returns an boolean if user found
*
*
- * @param user
- * user to find
- * @param newtoken
- * jwt token to use
+ * @param user user to find
+ * @param newtoken jwt token to use
* @return boolean true if found in userset
- * @throws CustomException
+ * @throws CustomException
*/
public boolean findUserInUserSet(String user, String newtoken) throws CustomException {
boolean found = false;
@@ -115,7 +108,7 @@ public boolean findUserInUserSet(String user, String newtoken) throws CustomExce
String apiUrl = this.apiUrlGetUsers + user;
apiUrl = removeQuotes(apiUrl);
-
+
try {
URL url = new URL(apiUrl);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
@@ -152,8 +145,8 @@ public boolean findUserInUserSet(String user, String newtoken) throws CustomExce
if (i > 0) {
found = true;
-
- }
+
+ }
connection.disconnect();
} catch (Exception e) {
if (e.getMessage().contains("403")) {
@@ -165,18 +158,16 @@ public boolean findUserInUserSet(String user, String newtoken) throws CustomExce
return found;
}
-
+
/**
- * Loads users from a file to the userset. Returns an int of number of users added.
+ * Loads users from a file to the userset. Returns an int of number of users
+ * added.
*
*
- * @param url
- * url to userset api
- * @param newtoken
- * jwt token to use
- * @param filePath
- * file to load
- * @return int totalnumberofrecords added to userset
+ * @param url url to userset api
+ * @param newtoken jwt token to use
+ * @param filePath file to load
+ * @return int totalnumberofrecords added to userset
*/
public int addAUserToUserSetFromFile(String url, String newtoken, String filePath) throws IOException {
@@ -191,7 +182,6 @@ public int addAUserToUserSetFromFile(String url, String newtoken, String filePat
while ((line = br.readLine()) != null) {
totalnbrofrecords++;
- // Append the email address to the payload
payloadBuilder.append("\"").append(line).append("\",");
count++;
@@ -221,20 +211,17 @@ public int addAUserToUserSetFromFile(String url, String newtoken, String filePat
}
/**
- * Loads users from a file to the userset. Returns an int of number of users added.
+ * Loads users from a file to the userset. Returns an int of number of users
+ * added.
*
*
- * @param payloadBuilder
- * payload for CM call that contains users to add
- * @param newtoken
- * jwt token to use
- * @param url
- * url to peform inserts
- * @return int response code
+ * @param payloadBuilder payload for CM call that contains users to add
+ * @param newtoken jwt token to use
+ * @param url url to peform inserts
+ * @return int response code
*/
-
- public static int makeCMCall(StringBuilder payloadBuilder, String newtoken, String url)
- throws IOException {
+
+ public static int makeCMCall(StringBuilder payloadBuilder, String newtoken, String url) throws IOException {
payloadBuilder.deleteCharAt(payloadBuilder.length() - 1); // Remove the trailing comma
payloadBuilder.append("]}");
@@ -288,15 +275,14 @@ public static int makeCMCall(StringBuilder payloadBuilder, String newtoken, Stri
}
/**
- * Simple sample of showing how to load a couple of users to the userset. Returns an int of number of users added.
+ * Simple sample of showing how to load a couple of users to the userset.
+ * Returns an int of number of users added.
*
*
- * @param url
- * url to userset api
- * @param newtoken
- * jwt token to use
- *
- * @return int response code
+ * @param url url to userset api
+ * @param newtoken jwt token to use
+ *
+ * @return int response code
*/
public static int addAUserToUserSet(String url, String newtoken) throws IOException {
boolean found = false;
@@ -367,15 +353,12 @@ private static int numberOfLines(RandomAccessFile file) throws IOException {
* Get JWT from CM
*
*
- * @param apiUrl
- * url to CM auth
- * @param username
- * username on CM
- * @param pwd
- * password on CM
- * @return String jwt
+ * @param apiUrl url to CM auth
+ * @param username username on CM
+ * @param pwd password on CM
+ * @return String jwt
*/
-
+
public static String geAuthToken(String apiUrl, String usernb, String pwd) throws Exception
{
@@ -383,7 +366,7 @@ public static String geAuthToken(String apiUrl, String usernb, String pwd) throw
String jStr = "{\"username\":\"" + usernb + "\",\"password\":\"" + pwd + "\"}";
disableCertValidation();
- String totalstr = null;
+ String jwtstr = null;
try {
URL url = new URL(apiUrl);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
@@ -403,38 +386,34 @@ public static String geAuthToken(String apiUrl, String usernb, String pwd) throw
}
reader.close();
-
JsonObject input = null;
- JsonElement total = null;
+ JsonElement jwt = null;
JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject();
if (rootNode.isJsonObject()) {
input = rootNode.getAsJsonObject();
if (input.isJsonObject()) {
- total = input.get("jwt");
+ jwt = input.get("jwt");
}
}
- JsonPrimitive column = total.getAsJsonPrimitive();
- totalstr = column.getAsJsonPrimitive().toString();
+ JsonPrimitive column = jwt.getAsJsonPrimitive();
+ jwtstr = column.getAsJsonPrimitive().toString();
connection.disconnect();
} catch (Exception e) {
e.printStackTrace();
}
- return totalstr;
+ return jwtstr;
}
-
+
/**
* Get JWT from CM
*
*
- * @param apiUrl
- * url to CM auth
- * @param username
- * username on CM
- * @param pwd
- * password on CM
- * @return String jwt
+ * @param apiUrl url to CM auth
+ * @param username username on CM
+ * @param pwd password on CM
+ * @return String jwt
*/
public static String geAuthToken(String apiUrl) throws Exception
@@ -447,7 +426,7 @@ public static String geAuthToken(String apiUrl) throws Exception
disableCertValidation();
- String totalstr = null;
+ String jwtstr = null;
try {
URL url = new URL(apiUrl);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
@@ -469,22 +448,22 @@ public static String geAuthToken(String apiUrl) throws Exception
if (debug)
System.out.println("response " + response);
JsonObject input = null;
- JsonElement total = null;
+ JsonElement jwt = null;
JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject();
if (rootNode.isJsonObject()) {
input = rootNode.getAsJsonObject();
if (input.isJsonObject()) {
- total = input.get("jwt");
+ jwt = input.get("jwt");
}
}
- JsonPrimitive column = total.getAsJsonPrimitive();
- totalstr = column.getAsJsonPrimitive().toString();
+ JsonPrimitive column = jwt.getAsJsonPrimitive();
+ jwtstr = column.getAsJsonPrimitive().toString();
connection.disconnect();
} catch (Exception e) {
e.printStackTrace();
}
- return totalstr;
+ return jwtstr;
}
diff --git a/database/redshift/src/main/java/example/ThalesAWSRedshiftCADPBulkFPE.java b/database/redshift/src/main/java/example/ThalesAWSRedshiftCADPBulkFPE.java
new file mode 100644
index 00000000..d62399ad
--- /dev/null
+++ b/database/redshift/src/main/java/example/ThalesAWSRedshiftCADPBulkFPE.java
@@ -0,0 +1,415 @@
+package example;
+
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.spec.AlgorithmParameterSpec;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Logger;
+import javax.crypto.Cipher;
+import org.apache.commons.io.IOUtils;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+import com.ingrian.security.nae.AbstractNAECipher;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec;
+import com.ingrian.security.nae.IngrianProvider;
+import com.ingrian.security.nae.NAEKey;
+import com.ingrian.security.nae.NAESession;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
+import com.ingrian.security.nae.IngrianProvider.Builder;
+import com.ingrian.security.nae.NAECipher;
+
+/*
+ * This test app to test the logic for a Redshift Database User Defined
+ * Function(UDF). It is an example of how to use Thales Cipher Trust Application Data Protection (CADP)
+ * to protect sensitive data in a column. This example uses
+ * Format Preserve Encryption (FPE) to maintain the original format of the data
+ * so applications or business intelligence tools do not have to change in order
+ * to use these columns.
+ *
+ * Note: This source code is only to be used for testing and proof of concepts.
+ * Not production ready code. Was not tested for all possible data sizes and
+ * combinations of encryption algorithms and IV, etc. Was tested with CM 2.14 &
+ * CADP 8.15.0.001 For more information on CADP see link below.
+* For more details on how to write Redshift UDF's please see
+* https://docs.aws.amazon.com/redshift/latest/dg/udf-creating-a-lambda-sql-udf.html#udf-lambda-json
+*
+Notes: This example uses the CADP bulk API.
+Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data
+element has Unicode characters then the supported size limit would be 1750 characters
+
+Size of spec array should be same as the number of elements if user wants to use separate spec values
+for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data
+index.
+ *
+ *@author mwarner
+ *
+ */
+
+public class ThalesAWSRedshiftCADPBulkFPE implements RequestStreamHandler {
+ private static final Logger logger = Logger.getLogger(ThalesAWSRedshiftCADPBulkFPE.class.getName());
+ private static final Gson gson = new Gson();
+
+ private static final String BADDATATAG = new String("9999999999999999");
+
+ private static byte[][] data;
+ private static AlgorithmParameterSpec[] spec;
+ private static int BATCHLIMIT = 2;
+
+
+ /**
+ * Returns an String that will be the encrypted value
+ *
+ * Examples: select thales_token_cadp_char(eventname) as enceventname ,
+ * eventname from event where len(eventname) > 5
+ *
+ * @param is any column in the database or any value that needs to be encrypted.
+ * Mostly used for ELT processes.
+ */
+
+ public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException {
+
+ String input = IOUtils.toString(inputStream, "UTF-8");
+ JsonParser parser = new JsonParser();
+ int statusCode = 200;
+
+ String redshiftreturnstring = null;
+ StringBuffer redshiftreturndata = new StringBuffer();
+
+ boolean status = true;
+ int numberofchunks = 0;
+ JsonObject redshiftinput = null;
+ JsonElement rootNode = parser.parse(input);
+ JsonArray redshiftdata = null;
+ String redshiftuserstr = null;
+
+ Map encryptedErrorMapTotal = new HashMap();
+ Map encryptedErrorMap = new HashMap();
+ // https://www.baeldung.com/java-aws-lambda
+
+ NAESession session = null;
+
+ if (rootNode.isJsonObject()) {
+ redshiftinput = rootNode.getAsJsonObject();
+ if (redshiftinput != null) {
+ redshiftdata = redshiftinput.getAsJsonArray("arguments");
+
+ JsonPrimitive userjson = redshiftinput.getAsJsonPrimitive("user");
+ redshiftuserstr = userjson.getAsJsonPrimitive().toString();
+ redshiftuserstr = redshiftuserstr.replace("\"", "");
+ } else {
+ System.out.println("Root node not found.");
+
+ }
+ } else {
+ System.out.println("Bad data from Redshift.");
+
+ }
+
+ JsonPrimitive nbr_of_rows_json = redshiftinput.getAsJsonPrimitive("num_records");
+ String nbr_of_rows_json_str = nbr_of_rows_json.getAsJsonPrimitive().toString();
+ int nbr_of_rows_json_int = new Integer(nbr_of_rows_json_str);
+
+ System.out.println("number of records " + nbr_of_rows_json_str);
+
+ // apiuser
+ String keyName = "testfaas";
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned
+ // when the user above does not have access to the key and if doing a
+ // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no
+ // then an error will be returned to the query, else the results set will provide ciphertext.
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ // yes,no
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB
+ // yes,no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be
+ // a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ int batchsize = Integer.parseInt(System.getenv("BATCHSIZE"));
+ int totalNbrofRows = redshiftdata.size();
+ int totalRowsLeft = totalNbrofRows;
+ String mode = System.getenv("mode");
+ String datatype = System.getenv("datatype");
+
+
+ if (batchsize > totalNbrofRows)
+ batchsize = totalNbrofRows;
+ if (batchsize >= BATCHLIMIT)
+ batchsize = BATCHLIMIT;
+
+
+ spec = new FPEParameterAndFormatSpec[batchsize];
+ data = new byte[batchsize][];
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+
+ try {
+
+ System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
+ "CADP_for_JAVA.properties");
+ IngrianProvider builder = new Builder().addConfigFileInputStream(
+ getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
+
+ session = NAESession.getSession(userName, password.toCharArray());
+ NAEKey key = NAEKey.getSecretKey(keyName, session);
+
+ int row_number = 0;
+
+ if (usersetlookupbool) {
+ // make sure cmuser is in Application Data Protection Clients Group
+
+ boolean founduserinuserset = findUserInUserSet(redshiftuserstr, userName, password, usersetID,
+ userSetLookupIP);
+ // System.out.println("Found User " + founduserinuserset);
+ if (!founduserinuserset)
+ throw new CustomException("1001, User Not in User Set", 1001);
+
+ } else {
+ usersetlookupbool = false;
+ }
+
+
+ int cipherType = 0;
+ String algorithm = "FPE/FF1v2/CARD62";
+
+ if (mode.equals("encrypt"))
+ cipherType = Cipher.ENCRYPT_MODE;
+ else
+ cipherType = Cipher.DECRYPT_MODE;
+
+ if (datatype.equals("char"))
+ algorithm = "FPE/FF1/CARD62";
+ else if (datatype.equals("charint"))
+ algorithm = "FPE/FF1/CARD10";
+ else
+ algorithm = "FPE/FF1/CARD10";
+
+ String tweakAlgo = null;
+ String tweakData = null;
+ FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
+ .build();
+
+ AbstractNAECipher thalesCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider");
+
+ thalesCipher.init(cipherType, key, spec[0]);
+
+ int i = 0;
+ int index = 0;
+ int count = 0;
+ boolean newchunk = true;
+ int dataIndex = 0;
+ int specIndex = 0;
+
+ while (i < totalNbrofRows) {
+ index = 0;
+
+ if (newchunk) {
+ if (totalRowsLeft < batchsize) {
+ spec = new FPEParameterAndFormatSpec[totalRowsLeft];
+ data = new byte[totalRowsLeft][];
+
+ } else {
+ spec = new FPEParameterAndFormatSpec[batchsize];
+ data = new byte[batchsize][];
+
+ }
+ newchunk = false;
+ }
+
+ JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
+
+ // insert new....
+ String sensitive = checkValid(redshiftrow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ sensitive = BADDATATAG+sensitive;
+ data[dataIndex++] = sensitive.getBytes();
+ // Can not return number since a leading 0 will not work.
+ // innerDataArray.add(new BigInteger(sensitive));
+ } else
+ data[dataIndex++] = BADDATATAG.getBytes();
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+ data[dataIndex++] = sensitive.getBytes();
+ } else {
+ data[dataIndex++] = sensitive.getBytes();
+ }
+ spec[specIndex++] = param;
+
+
+ } else {
+ data[dataIndex++] = sensitive.getBytes();
+ spec[specIndex++] = param;
+ }
+
+ if (count == batchsize - 1) {
+
+ // Map to store exceptions while encryption
+ encryptedErrorMap = new HashMap();
+
+ // performing bulk operation
+ byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap);
+
+ for (Map.Entry entry : encryptedErrorMap.entrySet()) {
+ Integer mkey = entry.getKey();
+ String mvalue = entry.getValue();
+ encryptedErrorMapTotal.put(mkey, mvalue);
+ }
+
+ for (int enc = 0; enc < encryptedData.length; enc++) {
+ dataArray.add(new String(encryptedData[enc]));
+ index++;
+
+ }
+
+ numberofchunks++;
+ newchunk = true;
+ count = 0;
+ dataIndex = 0;
+ specIndex = 0;
+ } else
+ count++;
+
+ totalRowsLeft--;
+ i++;
+ }
+
+ if (count > 0) {
+ numberofchunks++;
+ byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap);
+ for (int enc = 0; enc < encryptedData.length; enc++) {
+ dataArray.add(new String(encryptedData[enc]));
+ index++;
+ totalRowsLeft--;
+
+ }
+ }
+
+
+ bodyObject.addProperty("success", true);
+ bodyObject.addProperty("num_records", totalNbrofRows);
+ bodyObject.add("results", dataArray);
+ redshiftreturnstring = bodyObject.toString();
+
+
+ } catch (
+
+ Exception e) {
+ System.out.println("in exception with " + e.getMessage());
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401") || (e.getMessage().contains("1001") || (e.getMessage().contains("1002"))) ) {
+
+ for (int i = 0; i < redshiftdata.size(); i++) {
+ JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
+
+ JsonPrimitive redshiftcolumn = redshiftrow.get(0).getAsJsonPrimitive();
+
+ String sensitive = redshiftcolumn.getAsJsonPrimitive().toString();
+ dataArray.add(sensitive);
+ redshiftreturndata.append(sensitive);
+
+ }
+ bodyObject.addProperty("success", true);
+ bodyObject.addProperty("num_records", totalNbrofRows);
+ bodyObject.add("results", dataArray);
+ redshiftreturnstring = bodyObject.toString();
+
+ } else {
+ statusCode = 400;
+ redshiftreturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+
+ } else {
+ statusCode = 400;
+ redshiftreturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ }
+
+ finally {
+ if (session != null) {
+ session.closeSession();
+ }
+ }
+
+ //System.out.println("string = " + redshiftreturnstring);
+ System.out.println("numberofchunks = " + numberofchunks);
+ outputStream.write(new Gson().toJson(redshiftreturnstring).getBytes());
+ }
+
+ public String checkValid(JsonArray redshiftrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (redshiftrow != null && redshiftrow.size() > 0) {
+ JsonElement element = redshiftrow.get(0);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+
+ boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
+
+ return founduserinuserset;
+
+ }
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer redshiftreturndata = new StringBuffer();
+
+ String errormsg = "\"Error in UDF \"";
+ redshiftreturndata.append("{ \"success\":");
+ redshiftreturndata.append(false);
+ redshiftreturndata.append(" \"num_records\":");
+ redshiftreturndata.append(0);
+ redshiftreturndata.append(",");
+ redshiftreturndata.append(" \"error_msg\":");
+ redshiftreturndata.append(errormsg);
+ redshiftreturndata.append(",");
+ redshiftreturndata.append(" \"results\": [] }");
+
+ return redshiftreturndata.toString();
+ }
+
+}
\ No newline at end of file
diff --git a/database/redshift/src/main/java/example/ThalesAWSRedshiftCADPCharDecryptFPE.java b/database/redshift/src/main/java/example/ThalesAWSRedshiftCADPCharDecryptFPE.java
deleted file mode 100644
index 815b7336..00000000
--- a/database/redshift/src/main/java/example/ThalesAWSRedshiftCADPCharDecryptFPE.java
+++ /dev/null
@@ -1,284 +0,0 @@
-package example;
-
-import com.amazonaws.services.lambda.runtime.Context;
-import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.logging.Logger;
-import javax.crypto.Cipher;
-import javax.crypto.spec.IvParameterSpec;
-import org.apache.commons.io.IOUtils;
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-
-import com.google.gson.JsonParser;
-import com.google.gson.JsonPrimitive;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec;
-import com.ingrian.security.nae.IngrianProvider;
-import com.ingrian.security.nae.NAEKey;
-import com.ingrian.security.nae.NAESecureRandom;
-import com.ingrian.security.nae.NAESession;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
-import com.ingrian.security.nae.IngrianProvider.Builder;
-
-/*
- * This test app to test the logic for a Redshift Database User Defined
- * Function(UDF). It is an example of how to use Thales Cipher Trust Application Data Protection (CADP)
- * to protect sensitive data in a column. This example uses
- * Format Preserve Encryption (FPE) to maintain the original format of the data
- * so applications or business intelligence tools do not have to change in order
- * to use these columns. There is no need to deploy a function to run it.
- *
- * Note: This source code is only to be used for testing and proof of concepts.
- * Not production ready code. Was not tested for all possible data sizes and
- * combinations of encryption algorithms and IV, etc. Was tested with CM 2.14 &
- * CADP 8.15.0.001 For more information on CADP see link below.
-* For more details on how to write Redshift UDF's please see
-* https://docs.aws.amazon.com/redshift/latest/dg/udf-creating-a-lambda-sql-udf.html#udf-lambda-json
-*
- */
-
-public class ThalesAWSRedshiftCADPCharDecryptFPE implements RequestStreamHandler {
- private static final Logger logger = Logger.getLogger(ThalesAWSRedshiftCADPCharDecryptFPE.class.getName());
- private static final Gson gson = new Gson();
- /**
- * Returns an String that will be the encrypted value
- *
- * Examples:
- * select thales_token_cadp_char(eventname) as enceventname , eventname from event where len(eventname) > 5
- *
- * @param is any column in the database or any value that needs to be encrypted. Mostly used for ELT processes.
- */
-
- public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException {
-
-
-
- // context.getLogger().log("Input: " + inputStream);
- String input = IOUtils.toString(inputStream, "UTF-8");
- JsonParser parser = new JsonParser();
- NAESession session = null;
- int statusCode = 200;
-
- String redshiftreturnstring = null;
- StringBuffer redshiftreturndata = new StringBuffer();
-
- boolean status = true;
-
- JsonObject redshiftinput = null;
- JsonElement rootNode = parser.parse(input);
- JsonArray redshiftdata = null;
- String redshiftuserstr = null;
-
- if (rootNode.isJsonObject()) {
- redshiftinput = rootNode.getAsJsonObject();
- if (redshiftinput != null) {
- redshiftdata = redshiftinput.getAsJsonArray("arguments");
- JsonPrimitive userjson = redshiftinput.getAsJsonPrimitive("user");
- redshiftuserstr = userjson.getAsJsonPrimitive().toString();
- redshiftuserstr = redshiftuserstr.replace("\"", "");
- } else {
- System.out.println("Root node not found.");
-
- }
- } else {
- System.out.println("Bad data from snowflake.");
-
- }
-
- JsonPrimitive nbr_of_rows_json = redshiftinput.getAsJsonPrimitive("num_records");
- String nbr_of_rows_json_str = nbr_of_rows_json.getAsJsonPrimitive().toString();
- int nbr_of_rows_json_int = new Integer(nbr_of_rows_json_str);
-
- //System.out.println("number of records " + nbr_of_rows_json_str);
-
- String keyName = "testfaas";
- String userName = System.getenv("CMUSER");
- String password = System.getenv("CMPWD");
-
- // returnciphertextforuserwithnokeyaccess = is a environment variable to express
- // how data should be
- // returned when the user above does not have access to the key and if doing a
- // lookup in the userset
- // and the user does not exist. If returnciphertextforuserwithnokeyaccess = null
- // then an error will be
- // returned to the query, else the results set will provide ciphertext.
- // validvalues are 1 or null
- // 1 will return cipher text
- // null will return error.
- String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
- boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.matches("-?\\d+"); // Using regular
-
- // usersetlookup = should a userset lookup be done on the user from Big Query? 1
- // = true 0 = false.
- String usersetlookup = System.getenv("usersetlookup");
- // usersetID = should be the usersetid in CM to query.
- String usersetID = System.getenv("usersetidincm");
- // usersetlookupip = this is the IP address to query the userset. Currently it
- // is
- // the userset in CM but could be a memcache or other in memory db.
- String userSetLookupIP = System.getenv("usersetlookupip");
- boolean usersetlookupbool = usersetlookup.matches("-?\\d+");
-
- try {
-
- // Serialization
- redshiftreturndata.append("{ \"success\":");
- redshiftreturndata.append(status);
- redshiftreturndata.append(",");
- redshiftreturndata.append(" \"num_records\":");
- redshiftreturndata.append(nbr_of_rows_json_int);
- redshiftreturndata.append(",");
- redshiftreturndata.append(" \"results\": [");
-
- if (usersetlookupbool) {
- // Convert the string to an integer
- int num = Integer.parseInt(usersetlookup);
- // make sure cmuser is in Application Data Protection Clients Group
- if (num >= 1) {
- boolean founduserinuserset = true;
- try {
- founduserinuserset = findUserInUserSet(redshiftuserstr, userName, password, usersetID,
- userSetLookupIP);
- } catch (Exception e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- // System.out.println("Found User " + founduserinuserset);
- if (!founduserinuserset)
- throw new CustomException("1001, User Not in User Set", 1001);
-
- }
-
- else
- usersetlookupbool = false;
- } else {
- usersetlookupbool = false;
- }
-
- //System.setProperty("com.ingrian.security.nae.NAE_IP.1", "10.20.1.9");
- System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
- "CADP_for_JAVA.properties");
- IngrianProvider builder = new Builder().addConfigFileInputStream(
- getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
-
- session = NAESession.getSession(userName, password.toCharArray());
- NAEKey key = NAEKey.getSecretKey(keyName, session);
-
- String algorithm = "FPE/FF1/CARD62";
- // String algorithm = "AES/CBC/PKCS5Padding";
- String tweakAlgo = null;
- String tweakData = null;
- FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
- .build();
-
- Cipher decryptCipher = Cipher.getInstance(algorithm, "IngrianProvider");
- String encdata = "";
-
- for (int i = 0; i < redshiftdata.size(); i++) {
- JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
-
- JsonPrimitive redshiftcolumn = redshiftrow.get(0).getAsJsonPrimitive();
-
- String sensitive = redshiftcolumn.getAsJsonPrimitive().toString();
-
- // initialize cipher to decrypt.
- decryptCipher.init(Cipher.DECRYPT_MODE, key, param);
- // decrypt data
- byte[] outbuf = decryptCipher.doFinal(sensitive.getBytes());
- encdata = new String(outbuf);
-
- redshiftreturndata.append(encdata);
- if (redshiftdata.size() == 1 || i == redshiftdata.size() - 1)
- continue;
- else
- redshiftreturndata.append(",");
- }
-
- redshiftreturndata.append("]}");
-
- redshiftreturnstring = new String(redshiftreturndata);
-
-
- } catch (Exception e) {
- System.out.println("in exception with " + e.getMessage());
- if (returnciphertextbool) {
- if (e.getMessage().contains("1401") || (e.getMessage().contains("1001") || (e.getMessage().contains("1002"))) ) {
-
- for (int i = 0; i < redshiftdata.size(); i++) {
- JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
-
- JsonPrimitive redshiftcolumn = redshiftrow.get(0).getAsJsonPrimitive();
-
- String sensitive = redshiftcolumn.getAsJsonPrimitive().toString();
-
- redshiftreturndata.append(sensitive);
- if (redshiftdata.size() == 1 || i == redshiftdata.size() - 1)
- continue;
- else
- redshiftreturndata.append(",");
- }
- redshiftreturndata.append("]}");
-
- redshiftreturnstring = new String(redshiftreturndata);
-
- } else {
- statusCode = 400;
- redshiftreturnstring = formatReturnValue(statusCode);
- e.printStackTrace(System.out);
- }
-
- } else {
- statusCode = 400;
- redshiftreturnstring = formatReturnValue(statusCode);
- e.printStackTrace(System.out);
- }
-
- } finally {
- if (session != null) {
- session.closeSession();
- }
- }
- //System.out.println("string = " + redshiftreturnstring);
- outputStream.write(new Gson().toJson(redshiftreturnstring).getBytes());
-
- }
-
- public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
- String userSetLookupIP) throws Exception {
-
- CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
-
- String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
- String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
-
- boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
-
- return founduserinuserset;
-
- }
-
- public String formatReturnValue(int statusCode)
-
- {
- StringBuffer redshiftreturndata = new StringBuffer();
-
- String errormsg = "\"Error in UDF \"";
- redshiftreturndata.append("{ \"success\":");
- redshiftreturndata.append(false);
- redshiftreturndata.append(" \"num_records\":");
- redshiftreturndata.append(0);
- // redshiftreturndata.append(nbr_of_rows_json_int);
- redshiftreturndata.append(",");
- redshiftreturndata.append(" \"error_msg\":");
- redshiftreturndata.append(errormsg);
- redshiftreturndata.append(",");
- redshiftreturndata.append(" \"results\": [] }");
- // outputStream.write(redshiftreturnstring.getBytes());
-
- return redshiftreturndata.toString();
- }
-}
\ No newline at end of file
diff --git a/database/redshift/src/main/java/example/ThalesAWSRedshiftCADPCharEncryptBulkFPE.java b/database/redshift/src/main/java/example/ThalesAWSRedshiftCADPCharEncryptBulkFPE.java
deleted file mode 100644
index 91a52678..00000000
--- a/database/redshift/src/main/java/example/ThalesAWSRedshiftCADPCharEncryptBulkFPE.java
+++ /dev/null
@@ -1,259 +0,0 @@
-package example;
-
-import com.amazonaws.services.lambda.runtime.Context;
-import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.security.spec.AlgorithmParameterSpec;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.logging.Logger;
-import javax.crypto.Cipher;
-import org.apache.commons.io.IOUtils;
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-
-import com.google.gson.JsonParser;
-import com.google.gson.JsonPrimitive;
-import com.ingrian.security.nae.AbstractNAECipher;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec;
-import com.ingrian.security.nae.IngrianProvider;
-import com.ingrian.security.nae.NAEKey;
-import com.ingrian.security.nae.NAESession;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
-import com.ingrian.security.nae.IngrianProvider.Builder;
-import com.ingrian.security.nae.NAECipher;
-
-/* This sample Database User Defined Function(UDF) for AWS Redshift is an example of how to use Thales Cipher Trust Manager Protect Application
- * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
- * data so applications or business intelligence tools do not have to change in order to use these columns. This example encrypts data in a column or whatever
- * is passed to the function.
-*
-* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
-* for all possible data sizes and combinations of encryption algorithms and IV, etc.
-* Was tested with CM 2.11 & CADO 8.13 or above.
-* For more details on how to write Redshift UDF's please see
-* https://docs.aws.amazon.com/redshift/latest/dg/udf-creating-a-lambda-sql-udf.html#udf-lambda-json
-*
-Notes: This example uses the CADP bulk API.
-Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data
-element has Unicode characters then the supported size limit would be 1750 characters
-
-Size of spec array should be same as the number of elements if user wants to use separate spec values
-for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data
-index.
- *
- *@author mwarner
- *
- */
-
-public class ThalesAWSRedshiftCADPCharEncryptBulkFPE implements RequestStreamHandler {
- private static final Logger logger = Logger.getLogger(ThalesAWSRedshiftCADPCharEncryptBulkFPE.class.getName());
- private static final Gson gson = new Gson();
-
- private static byte[][] data;
- private static AlgorithmParameterSpec[] spec;
-
-
- private static int BATCHLIMIT = 10000;
-
-
- /**
- * Returns an String that will be the encrypted value
- *
- * Examples:
- * select thales_token_cadp_char(eventname) as enceventname , eventname from event where len(eventname) > 5
- *
- * @param is any column in the database or any value that needs to be encrypted. Mostly used for ELT processes.
- */
-
- public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException {
-
- String input = IOUtils.toString(inputStream, "UTF-8");
- JsonParser parser = new JsonParser();
-
- Map encryptedErrorMapTotal = new HashMap();
- String redshiftreturnstring = null;
- // https://www.baeldung.com/java-aws-lambda
-
- JsonObject redshiftinput = null;
- JsonElement rootNode = parser.parse(input);
- JsonArray redshiftdata = null;
- if (rootNode.isJsonObject()) {
- redshiftinput = rootNode.getAsJsonObject();
-
- redshiftdata = redshiftinput.getAsJsonArray("arguments");
- }
-
-
- StringBuffer redshiftreturndatasb = new StringBuffer();
- StringBuffer redshiftreturndatasc = new StringBuffer();
-
- boolean status = true;
- int nbr_of_rows_json_int = 0;
- NAESession session = null;
-
- try {
-
- String keyName = "testfaas";
- String userName = System.getenv("CMUSER");
- String password = System.getenv("CMPWD");
-
- int batchsize = Integer.parseInt(System.getenv("BATCHSIZE"));
- if (batchsize >= BATCHLIMIT)
- batchsize = BATCHLIMIT;
- spec = new FPEParameterAndFormatSpec[batchsize];
- data = new byte[batchsize][];
-
-
- JsonPrimitive nbr_of_rows_json = redshiftinput.getAsJsonPrimitive("num_records");
- String nbr_of_rows_json_str = nbr_of_rows_json.getAsJsonPrimitive().toString();
- nbr_of_rows_json_int = new Integer(nbr_of_rows_json_str);
-
- System.out.println("number of records " + nbr_of_rows_json_str);
-
- System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
- "CADP_for_JAVA.properties");
- IngrianProvider builder = new Builder().addConfigFileInputStream(
- getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
- session = NAESession.getSession(userName, password.toCharArray());
- NAEKey key = NAEKey.getSecretKey(keyName, session);
-
- int row_number = 0;
-
-
-
- // Serialization
- redshiftreturndatasb.append("{ \"success\":");
- redshiftreturndatasb.append(status);
- redshiftreturndatasb.append(",");
- redshiftreturndatasb.append(" \"num_records\":");
- redshiftreturndatasb.append(nbr_of_rows_json_int);
- redshiftreturndatasb.append(",");
- redshiftreturndatasb.append(" \"results\": [");
-
- String algorithm = "FPE/FF1/CARD62";
- // String algorithm = "AES/CBC/PKCS5Padding";
- String tweakAlgo = null;
- String tweakData = null;
-
- AbstractNAECipher encryptCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider");
- int i = 0;
-
- int totalRowsLeft = redshiftdata.size();
- //String encdata = "";
-
- while (i < redshiftdata.size()) {
- int index = 0;
- int dataIndex = 0;
- int specIndex = 0;
- int redshiftRowIndex = 0;
- //System.out.println("totalRowsLeft begining =" + totalRowsLeft);
- if (totalRowsLeft < batchsize) {
- spec = new FPEParameterAndFormatSpec[totalRowsLeft];
- data = new byte[totalRowsLeft][];
-
- } else {
- spec = new FPEParameterAndFormatSpec[batchsize];
- data = new byte[batchsize][];
- }
-
- for (int b = 0; b < batchsize && b < totalRowsLeft; b++) {
-
- JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
-
- for (int j = 0; j < redshiftrow.size(); j++) {
-
- JsonPrimitive redshiftcolumn = redshiftrow.get(j).getAsJsonPrimitive();
- System.out.print(redshiftcolumn + " ");
- String sensitive = redshiftcolumn.getAsJsonPrimitive().toString();
- // get a cipher
- // FPE example
- data[dataIndex++] = sensitive.getBytes();
- spec[specIndex++] = new FPEParameterAndFormatBuilder(tweakData)
- .set_tweakAlgorithm(tweakAlgo).build();
-
- }
-
- i++;
- }
- // make bulk call....
- // initializing the cipher for encrypt operation
- encryptCipher.init(Cipher.ENCRYPT_MODE, key, spec[0]);
-
- // Map to store exceptions while encryption
- Map encryptedErrorMap = new HashMap();
-
- // performing bulk operation
- byte[][] encryptedData = encryptCipher.doFinalBulk(data, spec, encryptedErrorMap);
-
- for (Map.Entry entry : encryptedErrorMap.entrySet()) {
- Integer mkey = entry.getKey();
- String mvalue = entry.getValue();
- encryptedErrorMapTotal.put(mkey, mvalue);
- }
-
- for (int enc = 0; enc < encryptedData.length; enc++) {
-
- redshiftreturndatasc.append("[");
- redshiftreturndatasc.append(new String(encryptedData[enc]));
-
- if (index <= batchsize - 1 || index < totalRowsLeft - 1)
- // if (index < batchsize -1 && index < totalRowsLeft -1 )
- {
- if (totalRowsLeft -1 > 0)
- redshiftreturndatasc.append("],");
- else
- redshiftreturndatasc.append("]");
- } else {
-
- redshiftreturndatasc.append("]");
- }
-
- index++;
- totalRowsLeft--;
-
- }
-
- // totalRowsLeft = redshiftdata.size() - i;
-
- }
-
- redshiftreturndatasc.append("] }");
- redshiftreturndatasb.append(redshiftreturndatasc);
- redshiftreturndatasb.append("}");
-
- redshiftreturnstring = new String(redshiftreturndatasb);
-
-
- } catch (
-
- Exception e) {
- status = false;
- String errormsg = "\"something went wrong, fix it\"";
- redshiftreturndatasb.append("{ \"success\":");
- redshiftreturndatasb.append(status);
- redshiftreturndatasb.append(" \"num_records\":");
- redshiftreturndatasb.append(0);
- // redshiftreturndata.append(nbr_of_rows_json_int);
- redshiftreturndatasb.append(",");
- redshiftreturndatasb.append(" \"error_msg\":");
- redshiftreturndatasb.append(errormsg);
- // redshiftreturndata.append(",");
- //redshiftreturndata.append(" \"results\": []");
- //outputStream.write(redshiftreturnstring.getBytes());
- System.out.println("in exception with ");
- e.printStackTrace(System.out);
- }
- finally{
- if(session!=null) {
- session.closeSession();
- }
- }
-
- outputStream.write(new Gson().toJson(redshiftreturnstring).getBytes());
- }
-}
\ No newline at end of file
diff --git a/database/redshift/src/main/java/example/ThalesAWSRedshiftCADPCharEncryptFPE.java b/database/redshift/src/main/java/example/ThalesAWSRedshiftCADPFPE.java
similarity index 55%
rename from database/redshift/src/main/java/example/ThalesAWSRedshiftCADPCharEncryptFPE.java
rename to database/redshift/src/main/java/example/ThalesAWSRedshiftCADPFPE.java
index ad8b17a9..eaf96f37 100644
--- a/database/redshift/src/main/java/example/ThalesAWSRedshiftCADPCharEncryptFPE.java
+++ b/database/redshift/src/main/java/example/ThalesAWSRedshiftCADPFPE.java
@@ -6,7 +6,10 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.util.logging.Logger;
+
+import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
import javax.crypto.spec.IvParameterSpec;
import org.apache.commons.io.IOUtils;
import com.google.gson.Gson;
@@ -35,37 +38,35 @@
* Note: This source code is only to be used for testing and proof of concepts.
* Not production ready code. Was not tested for all possible data sizes and
* combinations of encryption algorithms and IV, etc. Was tested with CM 2.14 &
- * CADP 8.15.0.001 For more information on CADP see link below.
+ * CADP 8.15.0.001 For more information on CADP see link below.
* For more details on how to write Redshift UDF's please see
* https://docs.aws.amazon.com/redshift/latest/dg/udf-creating-a-lambda-sql-udf.html#udf-lambda-json
*
*/
-public class ThalesAWSRedshiftCADPCharEncryptFPE implements RequestStreamHandler {
- private static final Logger logger = Logger.getLogger(ThalesAWSRedshiftCADPCharEncryptFPE.class.getName());
+public class ThalesAWSRedshiftCADPFPE implements RequestStreamHandler {
+ private static final Logger logger = Logger.getLogger(ThalesAWSRedshiftCADPFPE.class.getName());
private static final Gson gson = new Gson();
+ private static final String BADDATATAG = new String("9999999999999999");
/**
* Returns an String that will be the encrypted value
*
- * Examples: select thales_token_cadp_char(eventname) as enceventname ,
- * eventname from event where len(eventname) > 5
+ * Examples: select thales_token_cadp_char(eventname) as enceventname , eventname from event where len(eventname) >
+ * 5
*
- * @param is any column in the database or any value that needs to be encrypted.
- * Mostly used for ELT processes.
+ * @param is any column in the database or any value that needs to be encrypted. Mostly used for ELT processes.
*/
public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException {
// context.getLogger().log("Input: " + inputStream);
String input = IOUtils.toString(inputStream, "UTF-8");
+ // String input = inputStream;
JsonParser parser = new JsonParser();
NAESession session = null;
int statusCode = 200;
-
- String redshiftreturnstring = null;
- StringBuffer redshiftreturndata = new StringBuffer();
- boolean status = true;
+ String redshiftreturnstring = null;
JsonObject redshiftinput = null;
JsonElement rootNode = parser.parse(input);
@@ -76,6 +77,7 @@ public void handleRequest(InputStream inputStream, OutputStream outputStream, Co
redshiftinput = rootNode.getAsJsonObject();
if (redshiftinput != null) {
redshiftdata = redshiftinput.getAsJsonArray("arguments");
+
JsonPrimitive userjson = redshiftinput.getAsJsonPrimitive("user");
redshiftuserstr = userjson.getAsJsonPrimitive().toString();
redshiftuserstr = redshiftuserstr.replace("\"", "");
@@ -84,7 +86,7 @@ public void handleRequest(InputStream inputStream, OutputStream outputStream, Co
}
} else {
- System.out.println("Bad data from snowflake.");
+ System.out.println("Bad data from Redshift.");
}
@@ -92,73 +94,46 @@ public void handleRequest(InputStream inputStream, OutputStream outputStream, Co
String nbr_of_rows_json_str = nbr_of_rows_json.getAsJsonPrimitive().toString();
int nbr_of_rows_json_int = new Integer(nbr_of_rows_json_str);
- //System.out.println("number of records " + nbr_of_rows_json_str);
+ System.out.println("number of records " + nbr_of_rows_json_str);
String keyName = "testfaas";
String userName = System.getenv("CMUSER");
String password = System.getenv("CMPWD");
- // returnciphertextforuserwithnokeyaccess = is a environment variable to express
- // how data should be
- // returned when the user above does not have access to the key and if doing a
- // lookup in the userset
- // and the user does not exist. If returnciphertextforuserwithnokeyaccess = null
- // then an error will be
- // returned to the query, else the results set will provide ciphertext.
- // validvalues are 1 or null
- // 1 will return cipher text
- // null will return error.
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned
+ // when the user above does not have access to the key and if doing a
+ // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no
+ // then an error will be returned to the query, else the results set will provide ciphertext.
String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
- boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.matches("-?\\d+"); // Using regular
-
- // usersetlookup = should a userset lookup be done on the user from Big Query? 1
- // = true 0 = false.
+ // yes,no
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB
+ // yes,no
String usersetlookup = System.getenv("usersetlookup");
- // usersetID = should be the usersetid in CM to query.
+ // usersetidincm = should be the usersetid in CM to query.
String usersetID = System.getenv("usersetidincm");
- // usersetlookupip = this is the IP address to query the userset. Currently it
- // is
- // the userset in CM but could be a memcache or other in memory db.
+ // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be
+ // a memcache or other in memory db.
String userSetLookupIP = System.getenv("usersetlookupip");
- boolean usersetlookupbool = usersetlookup.matches("-?\\d+");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ String mode = System.getenv("mode");
+ String datatype = System.getenv("datatype");
try {
- // Serialization
- redshiftreturndata.append("{ \"success\":");
- redshiftreturndata.append(status);
- redshiftreturndata.append(",");
- redshiftreturndata.append(" \"num_records\":");
- redshiftreturndata.append(nbr_of_rows_json_int);
- redshiftreturndata.append(",");
- redshiftreturndata.append(" \"results\": [");
-
if (usersetlookupbool) {
- // Convert the string to an integer
- int num = Integer.parseInt(usersetlookup);
// make sure cmuser is in Application Data Protection Clients Group
- if (num >= 1) {
- boolean founduserinuserset = true;
- try {
- founduserinuserset = findUserInUserSet(redshiftuserstr, userName, password, usersetID,
- userSetLookupIP);
- } catch (Exception e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- // System.out.println("Found User " + founduserinuserset);
- if (!founduserinuserset)
- throw new CustomException("1001, User Not in User Set", 1001);
- }
+ boolean founduserinuserset = findUserInUserSet(redshiftuserstr, userName, password, usersetID,
+ userSetLookupIP);
+ // System.out.println("Found User " + founduserinuserset);
+ if (!founduserinuserset)
+ throw new CustomException("1001, User Not in User Set", 1001);
- else
- usersetlookupbool = false;
} else {
usersetlookupbool = false;
}
- //System.setProperty("com.ingrian.security.nae.NAE_IP.1", "10.20.1.9");
System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
"CADP_for_JAVA.properties");
IngrianProvider builder = new Builder().addConfigFileInputStream(
@@ -167,62 +142,50 @@ public void handleRequest(InputStream inputStream, OutputStream outputStream, Co
session = NAESession.getSession(userName, password.toCharArray());
NAEKey key = NAEKey.getSecretKey(keyName, session);
+ int cipherType = 0;
String algorithm = "FPE/FF1/CARD62";
- // String algorithm = "AES/CBC/PKCS5Padding";
+
+ if (mode.equals("encrypt"))
+ cipherType = Cipher.ENCRYPT_MODE;
+ else
+ cipherType = Cipher.DECRYPT_MODE;
+
+ if (datatype.equals("char"))
+ algorithm = "FPE/FF1/CARD62";
+ else if (datatype.equals("charint"))
+ algorithm = "FPE/FF1/CARD10";
+ else
+ algorithm = "FPE/FF1/CARD10";
+
String tweakAlgo = null;
String tweakData = null;
FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
.build();
- Cipher encryptCipher = Cipher.getInstance(algorithm, "IngrianProvider");
- String encdata = "";
-
- for (int i = 0; i < redshiftdata.size(); i++) {
- JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
+ Cipher thalesCipher = Cipher.getInstance(algorithm, "IngrianProvider");
- JsonPrimitive redshiftcolumn = redshiftrow.get(0).getAsJsonPrimitive();
+ // initialize cipher to encrypt.
+ thalesCipher.init(cipherType, key, param);
- String sensitive = redshiftcolumn.getAsJsonPrimitive().toString();
-
- // initialize cipher to encrypt.
- encryptCipher.init(Cipher.ENCRYPT_MODE, key, param);
- // encrypt data
- byte[] outbuf = encryptCipher.doFinal(sensitive.getBytes());
- encdata = new String(outbuf);
-
- redshiftreturndata.append(encdata);
- if (redshiftdata.size() == 1 || i == redshiftdata.size() - 1)
- continue;
- else
- redshiftreturndata.append(",");
- }
-
- redshiftreturndata.append("]}");
-
- redshiftreturnstring = new String(redshiftreturndata);
+ redshiftreturnstring = formatReturnValue(200, redshiftdata, false, thalesCipher, datatype);
} catch (Exception e) {
- System.out.println("in exception with " + e.getMessage());
- if (returnciphertextbool) {
- if (e.getMessage().contains("1401") || (e.getMessage().contains("1001") || (e.getMessage().contains("1002"))) ) {
-
- for (int i = 0; i < redshiftdata.size(); i++) {
- JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
- JsonPrimitive redshiftcolumn = redshiftrow.get(0).getAsJsonPrimitive();
-
- String sensitive = redshiftcolumn.getAsJsonPrimitive().toString();
+ System.out.println("In Exception with = " + e.getMessage());
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
- redshiftreturndata.append(sensitive);
- if (redshiftdata.size() == 1 || i == redshiftdata.size() - 1)
- continue;
- else
- redshiftreturndata.append(",");
+ try {
+ redshiftreturnstring = formatReturnValue(200, redshiftdata, true, null, datatype);
+ } catch (IllegalBlockSizeException e1) {
+ // TODO Auto-generated catch block
+ e1.printStackTrace();
+ } catch (BadPaddingException e1) {
+ // TODO Auto-generated catch block
+ e1.printStackTrace();
}
- redshiftreturndata.append("]}");
-
- redshiftreturnstring = new String(redshiftreturndata);
} else {
statusCode = 400;
@@ -241,7 +204,7 @@ public void handleRequest(InputStream inputStream, OutputStream outputStream, Co
session.closeSession();
}
}
-
+
outputStream.write(new Gson().toJson(redshiftreturnstring).getBytes());
}
@@ -260,6 +223,98 @@ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd,
}
+ public String checkValid(JsonArray redShiftRow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (redShiftRow != null && redShiftRow.size() > 0) {
+ JsonElement element = redShiftRow.get(0);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ public String formatReturnValue(int statusCode, JsonArray redShiftArray, boolean error, Cipher thalesCipher,
+ String datatype) throws IllegalBlockSizeException, BadPaddingException {
+ int row_number = 0;
+
+ String encdata = null;
+ String sensitive = null;
+ String formattedString = null;
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+ int nbrofrecords = redShiftArray.size();
+
+ for (int i = 0; i < nbrofrecords; i++) {
+ JsonArray redshiftRow = redShiftArray.get(i).getAsJsonArray();
+
+ sensitive = checkValid(redshiftRow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+
+ sensitive = sensitive.replace("notvalid", "");
+ dataArray.add(sensitive);
+ // Can not return number since a leading 0 will not work.
+ // innerDataArray.add(new BigInteger(sensitive));
+ } else
+ dataArray.add(BADDATATAG);
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+ dataArray.add("");
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ dataArray.add(sensitive);
+ } else {
+ dataArray.add(sensitive);
+ }
+
+ } else {
+ if (!error) {
+ byte[] outbuf = thalesCipher.doFinal(sensitive.getBytes());
+ encdata = new String(outbuf);
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ dataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ dataArray.add(encdata);
+ }
+ } else {
+ encdata = sensitive;
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ dataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ dataArray.add(encdata);
+ }
+ }
+ }
+
+ // innerDataArray.add(encdata);
+
+ }
+ bodyObject.addProperty("success", true);
+ bodyObject.addProperty("num_records", nbrofrecords);
+ bodyObject.add("results", dataArray);
+ formattedString = bodyObject.toString();
+ return formattedString;
+ // return bodyString;
+
+ }
+
public String formatReturnValue(int statusCode)
{
@@ -280,4 +335,5 @@ public String formatReturnValue(int statusCode)
return redshiftreturndata.toString();
}
+
}
\ No newline at end of file
diff --git a/database/redshift/src/main/java/example/ThalesAWSRedshiftCADPNbrDecryptFPE.java b/database/redshift/src/main/java/example/ThalesAWSRedshiftCADPNbrDecryptFPE.java
deleted file mode 100644
index 2b4a0968..00000000
--- a/database/redshift/src/main/java/example/ThalesAWSRedshiftCADPNbrDecryptFPE.java
+++ /dev/null
@@ -1,285 +0,0 @@
-package example;
-
-
-import com.amazonaws.services.lambda.runtime.Context;
-import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.logging.Logger;
-import javax.crypto.Cipher;
-import javax.crypto.spec.IvParameterSpec;
-import org.apache.commons.io.IOUtils;
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-
-import com.google.gson.JsonParser;
-import com.google.gson.JsonPrimitive;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec;
-import com.ingrian.security.nae.IngrianProvider;
-import com.ingrian.security.nae.NAEKey;
-import com.ingrian.security.nae.NAESecureRandom;
-import com.ingrian.security.nae.NAESession;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
-import com.ingrian.security.nae.IngrianProvider.Builder;
-
-/*
- * This test app to test the logic for a Redshift Database User Defined
- * Function(UDF). It is an example of how to use Thales Cipher Trust Application Data Protection (CADP)
- * to protect sensitive data in a column. This example uses
- * Format Preserve Encryption (FPE) to maintain the original format of the data
- * so applications or business intelligence tools do not have to change in order
- * to use these columns. There is no need to deploy a function to run it.
-*
-* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
-* for all possible data sizes and combinations of encryption algorithms and IV, etc.
-* Was tested with CM 2.11 & CADP 8.12.6 or above.
-* For more details on how to write Redshift UDF's please see
-* https://docs.aws.amazon.com/redshift/latest/dg/udf-creating-a-lambda-sql-udf.html#udf-lambda-json
-*
- */
-
-public class ThalesAWSRedshiftCADPNbrDecryptFPE implements RequestStreamHandler {
- private static final Logger logger = Logger.getLogger(ThalesAWSRedshiftCADPNbrDecryptFPE.class.getName());
- private static final Gson gson = new Gson();
- /**
- * Returns an String that will be the encrypted value
- *
- * Examples:
- * select thales_token_cadp_char(eventname) as enceventname , eventname from event where len(eventname) > 5
- *
- * @param is any column in the database or any value that needs to be encrypted. Mostly used for ELT processes.
- */
-
- public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException {
-
-
-
- // context.getLogger().log("Input: " + inputStream);
- String input = IOUtils.toString(inputStream, "UTF-8");
- JsonParser parser = new JsonParser();
- NAESession session = null;
- int statusCode = 200;
-
- String redshiftreturnstring = null;
- StringBuffer redshiftreturndata = new StringBuffer();
-
- boolean status = true;
-
- JsonObject redshiftinput = null;
- JsonElement rootNode = parser.parse(input);
- JsonArray redshiftdata = null;
- String redshiftuserstr = null;
-
- if (rootNode.isJsonObject()) {
- redshiftinput = rootNode.getAsJsonObject();
- if (redshiftinput != null) {
- redshiftdata = redshiftinput.getAsJsonArray("arguments");
- JsonPrimitive userjson = redshiftinput.getAsJsonPrimitive("user");
- redshiftuserstr = userjson.getAsJsonPrimitive().toString();
- redshiftuserstr = redshiftuserstr.replace("\"", "");
- } else {
- System.out.println("Root node not found.");
-
- }
- } else {
- System.out.println("Bad data from snowflake.");
-
- }
-
- JsonPrimitive nbr_of_rows_json = redshiftinput.getAsJsonPrimitive("num_records");
- String nbr_of_rows_json_str = nbr_of_rows_json.getAsJsonPrimitive().toString();
- int nbr_of_rows_json_int = new Integer(nbr_of_rows_json_str);
-
- //System.out.println("number of records " + nbr_of_rows_json_str);
-
- String keyName = "testfaas";
- String userName = System.getenv("CMUSER");
- String password = System.getenv("CMPWD");
-
- // returnciphertextforuserwithnokeyaccess = is a environment variable to express
- // how data should be
- // returned when the user above does not have access to the key and if doing a
- // lookup in the userset
- // and the user does not exist. If returnciphertextforuserwithnokeyaccess = null
- // then an error will be
- // returned to the query, else the results set will provide ciphertext.
- // validvalues are 1 or null
- // 1 will return cipher text
- // null will return error.
- String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
- boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.matches("-?\\d+"); // Using regular
-
- // usersetlookup = should a userset lookup be done on the user from Big Query? 1
- // = true 0 = false.
- String usersetlookup = System.getenv("usersetlookup");
- // usersetID = should be the usersetid in CM to query.
- String usersetID = System.getenv("usersetidincm");
- // usersetlookupip = this is the IP address to query the userset. Currently it
- // is
- // the userset in CM but could be a memcache or other in memory db.
- String userSetLookupIP = System.getenv("usersetlookupip");
- boolean usersetlookupbool = usersetlookup.matches("-?\\d+");
-
- try {
-
- // Serialization
- redshiftreturndata.append("{ \"success\":");
- redshiftreturndata.append(status);
- redshiftreturndata.append(",");
- redshiftreturndata.append(" \"num_records\":");
- redshiftreturndata.append(nbr_of_rows_json_int);
- redshiftreturndata.append(",");
- redshiftreturndata.append(" \"results\": [");
-
- if (usersetlookupbool) {
- // Convert the string to an integer
- int num = Integer.parseInt(usersetlookup);
- // make sure cmuser is in Application Data Protection Clients Group
- if (num >= 1) {
- boolean founduserinuserset = true;
- try {
- founduserinuserset = findUserInUserSet(redshiftuserstr, userName, password, usersetID,
- userSetLookupIP);
- } catch (Exception e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- // System.out.println("Found User " + founduserinuserset);
- if (!founduserinuserset)
- throw new CustomException("1001, User Not in User Set", 1001);
-
- }
-
- else
- usersetlookupbool = false;
- } else {
- usersetlookupbool = false;
- }
-
- //System.setProperty("com.ingrian.security.nae.NAE_IP.1", "10.20.1.9");
- System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
- "CADP_for_JAVA.properties");
- IngrianProvider builder = new Builder().addConfigFileInputStream(
- getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
-
- session = NAESession.getSession(userName, password.toCharArray());
- NAEKey key = NAEKey.getSecretKey(keyName, session);
-
- String algorithm = "FPE/FF1/CARD10";
- // String algorithm = "AES/CBC/PKCS5Padding";
- String tweakAlgo = null;
- String tweakData = null;
- FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
- .build();
-
- Cipher decryptCipher = Cipher.getInstance(algorithm, "IngrianProvider");
- String encdata = "";
-
- for (int i = 0; i < redshiftdata.size(); i++) {
- JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
-
- JsonPrimitive redshiftcolumn = redshiftrow.get(0).getAsJsonPrimitive();
-
- String sensitive = redshiftcolumn.getAsJsonPrimitive().toString();
-
- // initialize cipher to decrypt.
- decryptCipher.init(Cipher.DECRYPT_MODE, key, param);
- // decrypt data
- byte[] outbuf = decryptCipher.doFinal(sensitive.getBytes());
- encdata = new String(outbuf);
-
- redshiftreturndata.append(encdata);
- if (redshiftdata.size() == 1 || i == redshiftdata.size() - 1)
- continue;
- else
- redshiftreturndata.append(",");
- }
-
- redshiftreturndata.append("]}");
-
- redshiftreturnstring = new String(redshiftreturndata);
-
-
- } catch (Exception e) {
- System.out.println("in exception with " + e.getMessage());
- if (returnciphertextbool) {
- if (e.getMessage().contains("1401") || (e.getMessage().contains("1001") || (e.getMessage().contains("1002"))) ) {
-
- for (int i = 0; i < redshiftdata.size(); i++) {
- JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
-
- JsonPrimitive redshiftcolumn = redshiftrow.get(0).getAsJsonPrimitive();
-
- String sensitive = redshiftcolumn.getAsJsonPrimitive().toString();
-
- redshiftreturndata.append(sensitive);
- if (redshiftdata.size() == 1 || i == redshiftdata.size() - 1)
- continue;
- else
- redshiftreturndata.append(",");
- }
- redshiftreturndata.append("]}");
-
- redshiftreturnstring = new String(redshiftreturndata);
-
- } else {
- statusCode = 400;
- redshiftreturnstring = formatReturnValue(statusCode);
- e.printStackTrace(System.out);
- }
-
- } else {
- statusCode = 400;
- redshiftreturnstring = formatReturnValue(statusCode);
- e.printStackTrace(System.out);
- }
-
- } finally {
- if (session != null) {
- session.closeSession();
- }
- }
- //System.out.println("string = " + redshiftreturnstring);
- outputStream.write(new Gson().toJson(redshiftreturnstring).getBytes());
-
- }
-
- public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
- String userSetLookupIP) throws Exception {
-
- CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
-
- String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
- String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
-
- boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
-
- return founduserinuserset;
-
- }
-
- public String formatReturnValue(int statusCode)
-
- {
- StringBuffer redshiftreturndata = new StringBuffer();
-
- String errormsg = "\"Error in UDF \"";
- redshiftreturndata.append("{ \"success\":");
- redshiftreturndata.append(false);
- redshiftreturndata.append(" \"num_records\":");
- redshiftreturndata.append(0);
- // redshiftreturndata.append(nbr_of_rows_json_int);
- redshiftreturndata.append(",");
- redshiftreturndata.append(" \"error_msg\":");
- redshiftreturndata.append(errormsg);
- redshiftreturndata.append(",");
- redshiftreturndata.append(" \"results\": [] }");
- // outputStream.write(redshiftreturnstring.getBytes());
-
- return redshiftreturndata.toString();
- }
-
-}
\ No newline at end of file
diff --git a/database/redshift/src/main/java/example/ThalesAWSRedshiftCADPNbrEncryptFPE.java b/database/redshift/src/main/java/example/ThalesAWSRedshiftCADPNbrEncryptFPE.java
deleted file mode 100644
index 7c0e698b..00000000
--- a/database/redshift/src/main/java/example/ThalesAWSRedshiftCADPNbrEncryptFPE.java
+++ /dev/null
@@ -1,286 +0,0 @@
-package example;
-
-
-import com.amazonaws.services.lambda.runtime.Context;
-import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.logging.Logger;
-import javax.crypto.Cipher;
-import javax.crypto.spec.IvParameterSpec;
-import org.apache.commons.io.IOUtils;
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-
-import com.google.gson.JsonParser;
-import com.google.gson.JsonPrimitive;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec;
-import com.ingrian.security.nae.IngrianProvider;
-import com.ingrian.security.nae.NAEKey;
-import com.ingrian.security.nae.NAESecureRandom;
-import com.ingrian.security.nae.NAESession;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
-import com.ingrian.security.nae.IngrianProvider.Builder;
-
-
-/*
- * This test app to test the logic for a Redshift Database User Defined
- * Function(UDF). It is an example of how to use Thales Cipher Trust Application Data Protection (CADP)
- * to protect sensitive data in a column. This example uses
- * Format Preserve Encryption (FPE) to maintain the original format of the data
- * so applications or business intelligence tools do not have to change in order
- * to use these columns. There is no need to deploy a function to run it.
- *
- * Note: This source code is only to be used for testing and proof of concepts.
- * Not production ready code. Was not tested for all possible data sizes and
- * combinations of encryption algorithms and IV, etc. Was tested with CM 2.14 &
- * CADP 8.15.0.001 For more information on CADP see link below.
-* For more details on how to write Redshift UDF's please see
-* https://docs.aws.amazon.com/redshift/latest/dg/udf-creating-a-lambda-sql-udf.html#udf-lambda-json
-* These examples assume you have placed all of the protect app required jar files in the java lib directory of your project as specified in the Thales
-* protectapp documentation at: https://thalesdocs.com/ctp/cm/latest/
-*
- */
-
-public class ThalesAWSRedshiftCADPNbrEncryptFPE implements RequestStreamHandler {
- private static final Logger logger = Logger.getLogger(ThalesAWSRedshiftCADPNbrEncryptFPE.class.getName());
- private static final Gson gson = new Gson();
- /**
- * Returns an String that will be the encrypted value
- *
- * Examples:
- * select thales_token_cadp_char(eventname) as enceventname , eventname from event where len(eventname) > 5
- *
- * @param is any column in the database or any value that needs to be encrypted. Mostly used for ELT processes.
- */
-
- public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException {
- // context.getLogger().log("Input: " + inputStream);
- String input = IOUtils.toString(inputStream, "UTF-8");
- JsonParser parser = new JsonParser();
- NAESession session = null;
- int statusCode = 200;
-
- String redshiftreturnstring = null;
- StringBuffer redshiftreturndata = new StringBuffer();
-
- boolean status = true;
-
- JsonObject redshiftinput = null;
- JsonElement rootNode = parser.parse(input);
- JsonArray redshiftdata = null;
- String redshiftuserstr = null;
-
- if (rootNode.isJsonObject()) {
- redshiftinput = rootNode.getAsJsonObject();
- if (redshiftinput != null) {
- redshiftdata = redshiftinput.getAsJsonArray("arguments");
- JsonPrimitive userjson = redshiftinput.getAsJsonPrimitive("user");
- redshiftuserstr = userjson.getAsJsonPrimitive().toString();
- redshiftuserstr = redshiftuserstr.replace("\"", "");
- } else {
- System.out.println("Root node not found.");
-
- }
- } else {
- System.out.println("Bad data from snowflake.");
-
- }
-
- JsonPrimitive nbr_of_rows_json = redshiftinput.getAsJsonPrimitive("num_records");
- String nbr_of_rows_json_str = nbr_of_rows_json.getAsJsonPrimitive().toString();
- int nbr_of_rows_json_int = new Integer(nbr_of_rows_json_str);
-
- //System.out.println("number of records " + nbr_of_rows_json_str);
-
- String keyName = "testfaas";
- String userName = System.getenv("CMUSER");
- String password = System.getenv("CMPWD");
-
- // returnciphertextforuserwithnokeyaccess = is a environment variable to express
- // how data should be
- // returned when the user above does not have access to the key and if doing a
- // lookup in the userset
- // and the user does not exist. If returnciphertextforuserwithnokeyaccess = null
- // then an error will be
- // returned to the query, else the results set will provide ciphertext.
- // validvalues are 1 or null
- // 1 will return cipher text
- // null will return error.
- String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
- boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.matches("-?\\d+"); // Using regular
-
- // usersetlookup = should a userset lookup be done on the user from Big Query? 1
- // = true 0 = false.
- String usersetlookup = System.getenv("usersetlookup");
- // usersetID = should be the usersetid in CM to query.
- String usersetID = System.getenv("usersetidincm");
- // usersetlookupip = this is the IP address to query the userset. Currently it
- // is
- // the userset in CM but could be a memcache or other in memory db.
- String userSetLookupIP = System.getenv("usersetlookupip");
- boolean usersetlookupbool = usersetlookup.matches("-?\\d+");
-
- try {
-
- // Serialization
- redshiftreturndata.append("{ \"success\":");
- redshiftreturndata.append(status);
- redshiftreturndata.append(",");
- redshiftreturndata.append(" \"num_records\":");
- redshiftreturndata.append(nbr_of_rows_json_int);
- redshiftreturndata.append(",");
- redshiftreturndata.append(" \"results\": [");
-
- if (usersetlookupbool) {
- // Convert the string to an integer
- int num = Integer.parseInt(usersetlookup);
- // make sure cmuser is in Application Data Protection Clients Group
- if (num >= 1) {
- boolean founduserinuserset = true;
- try {
- founduserinuserset = findUserInUserSet(redshiftuserstr, userName, password, usersetID,
- userSetLookupIP);
- } catch (Exception e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- // System.out.println("Found User " + founduserinuserset);
- if (!founduserinuserset)
- throw new CustomException("1001, User Not in User Set", 1001);
-
- }
-
- else
- usersetlookupbool = false;
- } else {
- usersetlookupbool = false;
- }
-
- //System.setProperty("com.ingrian.security.nae.NAE_IP.1", "10.20.1.9");
- System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
- "CADP_for_JAVA.properties");
- IngrianProvider builder = new Builder().addConfigFileInputStream(
- getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
-
- session = NAESession.getSession(userName, password.toCharArray());
- NAEKey key = NAEKey.getSecretKey(keyName, session);
-
- String algorithm = "FPE/FF1/CARD10";
- // String algorithm = "AES/CBC/PKCS5Padding";
- String tweakAlgo = null;
- String tweakData = null;
- FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
- .build();
-
- Cipher encryptCipher = Cipher.getInstance(algorithm, "IngrianProvider");
- String encdata = "";
-
- for (int i = 0; i < redshiftdata.size(); i++) {
- JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
-
- JsonPrimitive redshiftcolumn = redshiftrow.get(0).getAsJsonPrimitive();
-
- String sensitive = redshiftcolumn.getAsJsonPrimitive().toString();
-
- // initialize cipher to encrypt.
- encryptCipher.init(Cipher.ENCRYPT_MODE, key, param);
- // encrypt data
- byte[] outbuf = encryptCipher.doFinal(sensitive.getBytes());
- encdata = new String(outbuf);
-
- redshiftreturndata.append(encdata);
- if (redshiftdata.size() == 1 || i == redshiftdata.size() - 1)
- continue;
- else
- redshiftreturndata.append(",");
- }
-
- redshiftreturndata.append("]}");
-
- redshiftreturnstring = new String(redshiftreturndata);
-
-
- } catch (Exception e) {
- System.out.println("in exception with " + e.getMessage());
- if (returnciphertextbool) {
- if (e.getMessage().contains("1401") || (e.getMessage().contains("1001") || (e.getMessage().contains("1002"))) ) {
-
- for (int i = 0; i < redshiftdata.size(); i++) {
- JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
-
- JsonPrimitive redshiftcolumn = redshiftrow.get(0).getAsJsonPrimitive();
-
- String sensitive = redshiftcolumn.getAsJsonPrimitive().toString();
-
- redshiftreturndata.append(sensitive);
- if (redshiftdata.size() == 1 || i == redshiftdata.size() - 1)
- continue;
- else
- redshiftreturndata.append(",");
- }
- redshiftreturndata.append("]}");
-
- redshiftreturnstring = new String(redshiftreturndata);
-
- } else {
- statusCode = 400;
- redshiftreturnstring = formatReturnValue(statusCode);
- e.printStackTrace(System.out);
- }
-
- } else {
- statusCode = 400;
- redshiftreturnstring = formatReturnValue(statusCode);
- e.printStackTrace(System.out);
- }
-
- } finally {
- if (session != null) {
- session.closeSession();
- }
- }
- //System.out.println("string = " + redshiftreturnstring);
- outputStream.write(new Gson().toJson(redshiftreturnstring).getBytes());
-
- }
-
- public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
- String userSetLookupIP) throws Exception {
-
- CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
-
- String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
- String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
-
- boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
-
- return founduserinuserset;
-
- }
-
- public String formatReturnValue(int statusCode)
-
- {
- StringBuffer redshiftreturndata = new StringBuffer();
-
- String errormsg = "\"Error in UDF \"";
- redshiftreturndata.append("{ \"success\":");
- redshiftreturndata.append(false);
- redshiftreturndata.append(" \"num_records\":");
- redshiftreturndata.append(0);
- // redshiftreturndata.append(nbr_of_rows_json_int);
- redshiftreturndata.append(",");
- redshiftreturndata.append(" \"error_msg\":");
- redshiftreturndata.append(errormsg);
- redshiftreturndata.append(",");
- redshiftreturndata.append(" \"results\": [] }");
- // outputStream.write(redshiftreturnstring.getBytes());
-
- return redshiftreturndata.toString();
- }
-
-}
\ No newline at end of file
diff --git a/database/redshift/src/main/java/example/ThalesAWSRedshiftCRDPBulkFPE.java b/database/redshift/src/main/java/example/ThalesAWSRedshiftCRDPBulkFPE.java
new file mode 100644
index 00000000..fcfd3922
--- /dev/null
+++ b/database/redshift/src/main/java/example/ThalesAWSRedshiftCRDPBulkFPE.java
@@ -0,0 +1,520 @@
+package example;
+
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.io.IOUtils;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+
+import okhttp3.MediaType;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+
+/*
+ * This test app to test the logic for a AWS Redshift Database User Defined
+ * Function(UDF). It is an example of how to use Thales Cipher REST Data Protection (CRDP)
+ * to protect sensitive data in a column. This example uses
+ * Format Preserve Encryption (FPE) to maintain the original format of the data
+ * so applications or business intelligence tools do not have to change in order
+ * to use these columns. There is no need to deploy a function to run it.
+ *
+ * This uses the protectbulk and revealbulk api's.
+ * Note: This source code is only to be used for testing and proof of concepts.
+ * Not production ready code. Was tested with CM 2.14 &
+ * CRDP 1.0 tech preview For more information on CRDP see link below.
+ * https://thalesdocs.com/ctp/con/crdp/latest/admin/index.html
+ *
+ * @author mwarner
+ *
+ */
+
+public class ThalesAWSRedshiftCRDPBulkFPE implements RequestStreamHandler {
+
+ private static final Gson gson = new Gson();
+
+ private static final String BADDATATAG = new String("9999999999999999");
+ private static int BATCHLIMIT = 10000;
+
+ private static final String REVEALRETURNTAG = new String("data");
+ private static final String PROTECTRETURNTAG = new String("protected_data");
+
+ /**
+ * Returns an String that will be the encrypted value
+ *
+ */
+
+ public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException {
+
+ String input = IOUtils.toString(inputStream, "UTF-8");
+
+ JsonParser parser = new JsonParser();
+ int statusCode = 200;
+
+ String redshiftreturnstring = null;
+ StringBuffer redshiftreturndata = new StringBuffer();
+
+ int numberofchunks = 0;
+ JsonObject redshiftinput = null;
+ JsonElement rootNode = parser.parse(input);
+ JsonArray redshiftdata = null;
+ String redshiftuserstr = null;
+
+ Map reshift_ErrorMap = new HashMap();
+ // https://www.baeldung.com/java-aws-lambda
+
+ StringBuffer protection_policy_buff = new StringBuffer();
+
+ JsonObject result = new JsonObject();
+ JsonArray replies = new JsonArray();
+
+ if (rootNode.isJsonObject()) {
+ redshiftinput = rootNode.getAsJsonObject();
+ if (redshiftinput != null) {
+ redshiftdata = redshiftinput.getAsJsonArray("arguments");
+ JsonPrimitive userjson = redshiftinput.getAsJsonPrimitive("user");
+ redshiftuserstr = userjson.getAsJsonPrimitive().toString();
+ redshiftuserstr = redshiftuserstr.replace("\"", "");
+ } else {
+ System.out.println("Root node not found.");
+ }
+ } else {
+ System.out.println("Bad data from snowflake.");
+ }
+
+ JsonPrimitive nbr_of_rows_json = redshiftinput.getAsJsonPrimitive("num_records");
+ String nbr_of_rows_json_str = nbr_of_rows_json.getAsJsonPrimitive().toString();
+ int nbr_of_rows_json_int = Integer.parseInt(nbr_of_rows_json_str);
+
+ String crdpip = System.getenv("CRDPIP");
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned
+ // when the user above does not have access to the key and if doing a
+ // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no
+ // then an error will be returned to the query, else the results set will provide ciphertext.
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ // yes,no
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB
+ // yes,no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be
+ // a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ String keymetadatalocation = System.getenv("keymetadatalocation");
+ String external_version_from_ext_source = System.getenv("keymetadata");
+ String protection_profile = System.getenv("protection_profile");
+ String mode = System.getenv("mode");
+ String datatype = System.getenv("datatype");
+
+ String inputDataKey = null;
+ String outputDataKey = null;
+ String protectedData = null;
+ String externalkeymetadata = null;
+ boolean bad_data = false;
+ String notvalid = "notvalid";
+ String jsonBody = null;
+ String jsonTagForProtectReveal = null;
+
+ int error_count = 0;
+
+ // String external_version_from_ext_source = "1004001";
+ // String external_version_from_ext_source = "1001001";
+
+ JsonObject crdp_payload = new JsonObject();
+
+ String showrevealkey = "yes";
+
+ if (mode.equals("protectbulk")) {
+ inputDataKey = "data_array";
+ outputDataKey = "protected_data_array";
+ jsonTagForProtectReveal = PROTECTRETURNTAG;
+ if (keymetadatalocation.equalsIgnoreCase("internal")) {
+ showrevealkey = System.getenv("showrevealinternalkey");
+ if (showrevealkey == null)
+ showrevealkey = "yes";
+ }
+ } else {
+ inputDataKey = "protected_data_array";
+ outputDataKey = "data_array";
+ jsonTagForProtectReveal = REVEALRETURNTAG;
+ }
+
+ boolean showrevealkeybool = showrevealkey.equalsIgnoreCase("yes");
+
+ int totalNbrofRows = redshiftdata.size();
+ int batchsize = Integer.parseInt(System.getenv("BATCHSIZE"));
+
+ if (batchsize > totalNbrofRows)
+ batchsize = totalNbrofRows;
+ if (batchsize >= BATCHLIMIT)
+ batchsize = BATCHLIMIT;
+
+ try {
+
+ int row_number = 0;
+
+ if (usersetlookupbool) {
+ // make sure cmuser is in Application Data Protection Clients Group
+
+ boolean founduserinuserset = findUserInUserSet(redshiftuserstr, userName, password, usersetID,
+ userSetLookupIP);
+ // System.out.println("Found User " + founduserinuserset);
+ if (!founduserinuserset)
+ throw new CustomException("1001, User Not in User Set", 1001);
+
+ } else {
+ usersetlookupbool = false;
+ }
+
+ int i = 0;
+ int count = 0;
+
+ JsonArray crdp_payload_array = new JsonArray();
+
+ OkHttpClient client = new OkHttpClient().newBuilder().build();
+ MediaType mediaType = MediaType.parse("application/json");
+ String urlStr = "http://" + crdpip + ":8090/v1/" + mode;
+
+ while (i < totalNbrofRows) {
+
+ String sensitive = null;
+ JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
+
+ sensitive = checkValid(redshiftrow);
+
+ protection_profile = protection_profile.trim();
+ // Format the output
+ String formattedElement = String.format("\"protection_policy_name\" : \"%s\"", protection_profile);
+ protection_policy_buff.append(formattedElement);
+ protection_policy_buff.append(",");
+
+ if (mode.equals("protectbulk")) {
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ sensitive = BADDATATAG + sensitive;
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ // sensitive = sensitive.replace("notvalid", "");
+
+ }
+
+ }
+
+ crdp_payload_array.add(sensitive);
+ } else {
+ JsonObject protectedDataObject = new JsonObject();
+ protectedDataObject.addProperty("protected_data", sensitive);
+ if (keymetadatalocation.equalsIgnoreCase("external")) {
+ protectedDataObject.addProperty("external_version", external_version_from_ext_source);
+ }
+ crdp_payload_array.add(protectedDataObject);
+
+ }
+ if (count == batchsize - 1) {
+
+ crdp_payload.add(inputDataKey, crdp_payload_array);
+ String inputdataarray = null;
+ if (mode.equals("revealbulk")) {
+ crdp_payload.addProperty("username", redshiftuserstr);
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ jsonBody = protection_policy_buff.toString();
+ jsonBody = jsonBody.replaceFirst("\\{", " ");
+
+ } else {
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ inputdataarray = protection_policy_buff.toString();
+ jsonBody = inputdataarray.replace("{", " ");
+ }
+ jsonBody = "{" + jsonBody;
+
+ RequestBody body = RequestBody.create(mediaType, jsonBody);
+
+ Request crdp_request = new Request.Builder().url(urlStr).method("POST", body)
+ .addHeader("Content-Type", "application/json").build();
+ Response crdp_response = client.newCall(crdp_request).execute();
+
+ if (crdp_response.isSuccessful()) {
+ // Parse JSON response
+ String responseBody = crdp_response.body().string();
+ JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class);
+ JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey);
+
+ String status = jsonObject.get("status").getAsString();
+ int success_count = jsonObject.get("success_count").getAsInt();
+ error_count = jsonObject.get("error_count").getAsInt();
+ if (error_count > 0)
+ System.out.println("errors " + error_count);
+ for (JsonElement element : protectedDataArray) {
+
+ JsonObject protectedDataObject = element.getAsJsonObject();
+ if (protectedDataObject.has(jsonTagForProtectReveal)) {
+
+ protectedData = protectedDataObject.get(jsonTagForProtectReveal).getAsString();
+
+ if (keymetadatalocation.equalsIgnoreCase("internal")
+ && mode.equalsIgnoreCase("protectbulk") && !showrevealkeybool) {
+ if (protectedData.length() > 7)
+ protectedData = protectedData.substring(7);
+ }
+
+ replies.add(new String(protectedData));
+
+ if (mode.equals("protectbulk")) {
+ if (keymetadatalocation.equalsIgnoreCase("external")
+ && mode.equalsIgnoreCase("protectbulk")) {
+ externalkeymetadata = protectedDataObject.get("external_version").getAsString();
+ // System.out.println("Protected Data ext key metadata need to store this: "
+ // + externalkeymetadata);
+
+ }
+ }
+ } else if (protectedDataObject.has("error_message")) {
+ String errorMessage = protectedDataObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ reshift_ErrorMap.put(i, errorMessage);
+ bad_data = true;
+ } else
+ System.out.println("unexpected json value from results: ");
+
+ }
+
+ crdp_response.close();
+ crdp_payload_array = new JsonArray();
+ protection_policy_buff = new StringBuffer();
+ numberofchunks++;
+
+ count = 0;
+ } else {
+ System.err.println("Request failed with status code: " + crdp_response.code());
+ }
+ } else {
+ count++;
+ }
+
+ // totalRowsLeft--;
+ i++;
+
+ }
+
+ if (count > 0) {
+
+ crdp_payload.add(inputDataKey, crdp_payload_array);
+ String inputdataarray = null;
+ if (mode.equals("revealbulk")) {
+ crdp_payload.addProperty("username", redshiftuserstr);
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ jsonBody = protection_policy_buff.toString();
+ jsonBody = jsonBody.replaceFirst("\\{", " ");
+
+ } else {
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ inputdataarray = protection_policy_buff.toString();
+ jsonBody = inputdataarray.replace("{", " ");
+ }
+ jsonBody = "{" + jsonBody;
+
+ RequestBody body = RequestBody.create(mediaType, jsonBody);
+
+ Request crdp_request = new Request.Builder().url(urlStr).method("POST", body)
+ .addHeader("Content-Type", "application/json").build();
+ Response crdp_response = client.newCall(crdp_request).execute();
+ String crdpreturnstr = null;
+ if (crdp_response.isSuccessful()) {
+ // Parse JSON response
+ String responseBody = crdp_response.body().string();
+ JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class);
+ JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey);
+
+ String status = jsonObject.get("status").getAsString();
+ int success_count = jsonObject.get("success_count").getAsInt();
+ error_count = jsonObject.get("error_count").getAsInt();
+ if (error_count > 0)
+ System.out.println("errors " + error_count);
+ for (JsonElement element : protectedDataArray) {
+
+ JsonObject protectedDataObject = element.getAsJsonObject();
+ if (protectedDataObject.has(jsonTagForProtectReveal)) {
+
+ protectedData = protectedDataObject.get(jsonTagForProtectReveal).getAsString();
+ if (keymetadatalocation.equalsIgnoreCase("internal") && mode.equalsIgnoreCase("protectbulk")
+ && !showrevealkeybool) {
+ if (protectedData.length() > 7)
+ protectedData = protectedData.substring(7);
+ }
+ replies.add(new String(protectedData));
+
+ if (mode.equals("protectbulk")) {
+ if (keymetadatalocation.equalsIgnoreCase("external")
+ && mode.equalsIgnoreCase("protectbulk")) {
+ externalkeymetadata = protectedDataObject.get("external_version").getAsString();
+ // System.out.println("Protected Data ext key metadata need to store this: "
+ // + externalkeymetadata);
+
+ }
+ }
+ } else if (protectedDataObject.has("error_message")) {
+ String errorMessage = protectedDataObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ reshift_ErrorMap.put(i, errorMessage);
+ bad_data = true;
+ } else
+ System.out.println("unexpected json value from results: ");
+
+ }
+
+ crdp_response.close();
+ crdp_payload_array = new JsonArray();
+ protection_policy_buff = new StringBuffer();
+ numberofchunks++;
+ count = 0;
+ } else {
+ System.err.println("Request failed with status code: " + crdp_response.code());
+ }
+ }
+
+ result.addProperty("success", true);
+ result.addProperty("num_records", totalNbrofRows);
+ result.add("results", replies);
+ redshiftreturnstring = result.toString();
+
+ } catch (
+
+ Exception e) {
+ System.out.println("in exception with " + e.getMessage());
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
+
+ for (int i = 0; i < redshiftdata.size(); i++) {
+ JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
+
+ JsonPrimitive redshiftcolumn = redshiftrow.get(0).getAsJsonPrimitive();
+
+ String sensitive = redshiftcolumn.getAsJsonPrimitive().toString();
+ replies.add(sensitive);
+ redshiftreturndata.append(sensitive);
+
+ }
+ result.addProperty("success", true);
+ result.addProperty("num_records", totalNbrofRows);
+ result.add("results", replies);
+ redshiftreturnstring = result.toString();
+
+ } else {
+ statusCode = 400;
+ redshiftreturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+
+ } else {
+ statusCode = 400;
+ redshiftreturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ }
+
+ finally {
+
+ }
+
+ if (bad_data) {
+ System.out.println("errors: ");
+ for (Map.Entry entry : reshift_ErrorMap.entrySet()) {
+ Integer mkey = entry.getKey();
+ String mvalue = entry.getValue();
+ System.out.println("Error index: " + mkey);
+ System.out.println("Error Message: " + mvalue);
+ }
+
+ }
+
+ System.out.println("numberofchunks = " + numberofchunks);
+
+ outputStream.write(new Gson().toJson(redshiftreturnstring).getBytes());
+ }
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+
+ boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
+
+ return founduserinuserset;
+
+ }
+
+ public String checkValid(JsonArray redshiftrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (redshiftrow != null && redshiftrow.size() > 0) {
+ JsonElement element = redshiftrow.get(0);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer redshiftreturndata = new StringBuffer();
+
+ String errormsg = "\"Error in UDF \"";
+ redshiftreturndata.append("{ \"success\":");
+ redshiftreturndata.append(false);
+ redshiftreturndata.append(" \"num_records\":");
+ redshiftreturndata.append(0);
+ redshiftreturndata.append(",");
+ redshiftreturndata.append(" \"error_msg\":");
+ redshiftreturndata.append(errormsg);
+ redshiftreturndata.append(",");
+ redshiftreturndata.append(" \"results\": [] }");
+
+ return redshiftreturndata.toString();
+ }
+
+}
\ No newline at end of file
diff --git a/database/redshift/src/main/java/example/ThalesAWSRedshiftCRDPFPE.java b/database/redshift/src/main/java/example/ThalesAWSRedshiftCRDPFPE.java
new file mode 100644
index 00000000..ef49d4e2
--- /dev/null
+++ b/database/redshift/src/main/java/example/ThalesAWSRedshiftCRDPFPE.java
@@ -0,0 +1,371 @@
+package example;
+
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Logger;
+
+import org.apache.commons.io.IOUtils;
+
+import okhttp3.MediaType;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+
+/*
+ * This test app to test the logic for a AWS Redshift Database User Defined
+ * Function(UDF). It is an example of how to use Thales Cipher REST Data Protection (CRDP)
+ * to protect sensitive data in a column. This example uses
+ * Format Preserve Encryption (FPE) to maintain the original format of the data
+ * so applications or business intelligence tools do not have to change in order
+ * to use these columns. There is no need to deploy a function to run it.
+ *
+ * Note: This source code is only to be used for testing and proof of concepts.
+ * Not production ready code. Was tested with CM 2.14 &
+ * CRDP 1.0 tech preview For more information on CRDP see link below.
+ * https://thalesdocs.com/ctp/con/crdp/latest/admin/index.html
+ *
+ * @author mwarner
+ *
+ */
+
+public class ThalesAWSRedshiftCRDPFPE implements RequestStreamHandler {
+ private static final Logger logger = Logger.getLogger(ThalesAWSRedshiftCRDPFPE.class.getName());
+
+ private static final String BADDATATAG = new String("9999999999999999");
+
+ private static final String REVEALRETURNTAG = new String("data");
+ private static final String PROTECTRETURNTAG = new String("protected_data");
+
+ private static final Gson gson = new Gson();
+
+ public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException {
+ Map reshift_ErrorMap = new HashMap();
+ String input = IOUtils.toString(inputStream, "UTF-8");
+ JsonParser parser = new JsonParser();
+
+ int statusCode = 200;
+
+ String encdata = "";
+ String redshiftreturnstring = null;
+ String sensitive = null;
+
+ boolean status = true;
+
+ JsonObject redshiftinput = null;
+ JsonElement rootNode = parser.parse(input);
+ JsonArray redshiftdata = null;
+ String redshiftuserstr = null;
+
+ if (rootNode.isJsonObject()) {
+ redshiftinput = rootNode.getAsJsonObject();
+ if (redshiftinput != null) {
+ redshiftdata = redshiftinput.getAsJsonArray("arguments");
+ JsonPrimitive userjson = redshiftinput.getAsJsonPrimitive("user");
+ redshiftuserstr = userjson.getAsJsonPrimitive().toString();
+ redshiftuserstr = redshiftuserstr.replace("\"", "");
+ } else {
+ System.out.println("Root node not found.");
+ }
+ } else {
+ System.out.println("Bad data from snowflake.");
+ }
+
+ JsonPrimitive nbr_of_rows_json = redshiftinput.getAsJsonPrimitive("num_records");
+ String nbr_of_rows_json_str = nbr_of_rows_json.getAsJsonPrimitive().toString();
+ int nbr_of_rows_json_int = Integer.parseInt(nbr_of_rows_json_str);
+
+ String crdpip = System.getenv("CRDPIP");
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned
+ // when the user above does not have access to the key and if doing a
+ // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no
+ // then an error will be returned to the query, else the results set will provide ciphertext.
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ // yes,no
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB
+ // yes,no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be
+ // a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ ;
+ String keymetadatalocation = System.getenv("keymetadatalocation");
+ String external_version_from_ext_source = System.getenv("keymetadata");
+ String protection_profile = System.getenv("protection_profile");
+ String mode = System.getenv("mode");
+ String datatype = System.getenv("datatype");
+
+ String dataKey = null;
+ String protectedData = null;
+ String externalkeymetadata = null;
+ String crdpjsonBody = null;
+ boolean bad_data = false;
+
+ JsonObject result = new JsonObject();
+ JsonArray replies = new JsonArray();
+
+ // String external_version_from_ext_source = "1004001";
+ // String external_version_from_ext_source = "1001001";
+
+ JsonObject crdp_payload = new JsonObject();
+
+ crdp_payload.addProperty("protection_policy_name", protection_profile);
+ String jsonTagForProtectReveal = null;
+
+ String showrevealkey = "yes";
+
+ if (mode.equals("protect")) {
+ dataKey = "data";
+ jsonTagForProtectReveal = PROTECTRETURNTAG;
+ if (keymetadatalocation.equalsIgnoreCase("internal")) {
+ showrevealkey = System.getenv("showrevealinternalkey");
+ if (showrevealkey == null)
+ showrevealkey = "yes";
+ }
+ } else {
+ dataKey = "protected_data";
+ jsonTagForProtectReveal = REVEALRETURNTAG;
+ }
+ boolean showrevealkeybool = showrevealkey.equalsIgnoreCase("yes");
+
+ int nbrofrecords = redshiftdata.size();
+
+ try {
+
+ if (usersetlookupbool) {
+ // make sure cmuser is in Application Data Protection Clients Group
+
+ boolean founduserinuserset = findUserInUserSet(redshiftuserstr, userName, password, usersetID,
+ userSetLookupIP);
+ // System.out.println("Found User " + founduserinuserset);
+ if (!founduserinuserset)
+ throw new CustomException("1001, User Not in User Set", 1001);
+
+ } else {
+ usersetlookupbool = false;
+ }
+ OkHttpClient client = new OkHttpClient().newBuilder().build();
+ MediaType mediaType = MediaType.parse("application/json");
+ String urlStr = "http://" + crdpip + ":8090/v1/" + mode;
+
+ for (int i = 0; i < nbrofrecords; i++) {
+ JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
+
+ sensitive = checkValid(redshiftrow);
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+
+ }
+ encdata = sensitive;
+ } else {
+
+ crdp_payload.addProperty(dataKey, sensitive);
+ if (mode.equals("reveal")) {
+ crdp_payload.addProperty("username", redshiftuserstr);
+ if (keymetadatalocation.equalsIgnoreCase("external")) {
+ crdp_payload.addProperty("external_version", external_version_from_ext_source);
+ }
+ }
+ crdpjsonBody = crdp_payload.toString();
+
+ RequestBody body = RequestBody.create(mediaType, crdpjsonBody);
+
+ Request crdp_request = new Request.Builder()
+ // .url("http://192.168.159.143:8090/v1/protect").method("POST", body)
+ .url(urlStr).method("POST", body).addHeader("Content-Type", "application/json").build();
+ Response crdp_response = client.newCall(crdp_request).execute();
+
+ if (crdp_response.isSuccessful()) {
+
+ // Parse JSON response
+ String responseBody = crdp_response.body().string();
+ Gson gson = new Gson();
+ JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class);
+
+ if (jsonObject.has(jsonTagForProtectReveal)) {
+ protectedData = jsonObject.get(jsonTagForProtectReveal).getAsString();
+ if (keymetadatalocation.equalsIgnoreCase("external") && mode.equalsIgnoreCase("protect")) {
+ externalkeymetadata = jsonObject.get("external_version").getAsString();
+ // System.out.println(
+ // "Protected Data ext key metadata need to store this: " + externalkeymetadata);
+ }
+
+ if (keymetadatalocation.equalsIgnoreCase("internal") && mode.equalsIgnoreCase("protect")
+ && !showrevealkeybool) {
+ if (protectedData.length() > 7)
+ protectedData = protectedData.substring(7);
+ }
+
+ } else if (jsonObject.has("error_message")) {
+ String errorMessage = jsonObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ reshift_ErrorMap.put(i, errorMessage);
+ bad_data = true;
+ } else
+ System.out.println("unexpected json value from results: ");
+
+ } else {
+ System.err.println("Request failed with status code: " + crdp_response.code());
+ }
+
+ crdp_response.close();
+
+ encdata = protectedData;
+
+ }
+ replies.add(encdata);
+ } // end for loop
+
+ result.addProperty("success", true);
+ result.addProperty("num_records", nbrofrecords);
+ result.add("results", replies);
+
+ redshiftreturnstring = result.toString();
+
+ } catch (Exception e) {
+ System.out.println("in exception with " + e.getMessage());
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
+
+ for (int i = 0; i < redshiftdata.size(); i++) {
+ JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
+
+ JsonPrimitive redshiftcolumn = redshiftrow.get(0).getAsJsonPrimitive();
+
+ sensitive = redshiftcolumn.getAsJsonPrimitive().toString();
+ replies.add(sensitive);
+
+ }
+ result.addProperty("success", true);
+ result.addProperty("num_records", nbrofrecords);
+ result.add("results", replies);
+ redshiftreturnstring = result.toString();
+
+ } else {
+ statusCode = 400;
+ redshiftreturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+
+ } else {
+ statusCode = 400;
+ redshiftreturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ }
+
+ finally {
+
+ }
+
+ if (bad_data) {
+ System.out.println("errors: ");
+ for (Map.Entry entry : reshift_ErrorMap.entrySet()) {
+ Integer mkey = entry.getKey();
+ String mvalue = entry.getValue();
+ System.out.println("Error index: " + mkey);
+ System.out.println("Error Message: " + mvalue);
+ }
+
+ }
+
+ // System.out.println(redshiftreturnstring);
+ outputStream.write(new Gson().toJson(redshiftreturnstring).getBytes());
+ }
+
+ public String checkValid(JsonArray redshiftrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (redshiftrow != null && redshiftrow.size() > 0) {
+ JsonElement element = redshiftrow.get(0);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ public static String formatString(String inputString) {
+ // Split the input string to isolate the array content
+ String[] parts = inputString.split("\\[")[1].split("\\]")[0].split(",");
+
+ // Reformat the array elements to enclose them within double quotes
+ StringBuilder formattedArray = new StringBuilder();
+ for (String part : parts) {
+ formattedArray.append("\"").append(part.trim()).append("\",");
+ }
+
+ // Build the final formatted string
+ return inputString.replaceFirst("\\[.*?\\]",
+ "[" + formattedArray.deleteCharAt(formattedArray.length() - 1) + "]");
+ }
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer redshiftreturndata = new StringBuffer();
+
+ String errormsg = "\"Error in UDF \"";
+ redshiftreturndata.append("{ \"success\":");
+ redshiftreturndata.append(false);
+ redshiftreturndata.append(" \"num_records\":");
+ redshiftreturndata.append(0);
+ redshiftreturndata.append(",");
+ redshiftreturndata.append(" \"error_msg\":");
+ redshiftreturndata.append(errormsg);
+ redshiftreturndata.append(",");
+ redshiftreturndata.append(" \"results\": [] }");
+
+ return redshiftreturndata.toString();
+ }
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+ return cmuserset.findUserInUserSet(userName, newtoken);
+ }
+
+}
diff --git a/database/redshift/src/main/python/thales_ct-vl-lambda_function_tokenize.py b/database/redshift/src/main/python/thales_ct-vl-lambda_function_tokenize.py
new file mode 100644
index 00000000..daf1106b
--- /dev/null
+++ b/database/redshift/src/main/python/thales_ct-vl-lambda_function_tokenize.py
@@ -0,0 +1,95 @@
+import os
+import sys
+import requests
+import json
+from urllib3 import disable_warnings
+from urllib3.exceptions import InsecureRequestWarning
+disable_warnings(InsecureRequestWarning)
+
+HTTP_SUCCESS = 200
+HTTP_FAILURE = 400
+CTSNBRTOKENGRP = "nbr"
+CTSNBRTOKENTEMP = "nbr"
+CTSCHARDIGITTOKENTEMP = "chardigit"
+CTSCHARALPHATOKENTEMP = "charalpha"
+CTSCHARALPHATOKENGRP = "char"
+# The following are enviroment variables.
+
+ctsuser = os.environ.get('CTSUSER', 'cts-user')
+ctspwd = os.environ.get('CTSPWD','Yourpwd')
+ctsip = os.environ.get('CTSIP', 'YourCTSIP')
+
+p11debug = False
+p11user = ctsuser + ":" + ctspwd
+p11url = os.environ.get('CTSIP', 'YourCTSIP')
+p11tokgroup = CTSCHARALPHATOKENGRP
+p11toktemplate = CTSCHARALPHATOKENTEMP
+p11auth = None
+
+p11url = "https://" + p11url + "/vts"
+# split the p11user variable to check username/password missing
+p11auth = tuple(p11user.split(":", 1))
+
+errs = 0
+
+# Set the header application
+hdrs = {'Content-Type': 'application/json'}
+
+############ Check connect to VTS before continue #######################
+try:
+ r = requests.get(p11url, verify=False)
+ r.raise_for_status()
+except requests.exceptions.RequestException as err:
+ print("\nEror, something wrong: ")
+ print(err)
+ sys.exit(HTTP_FAILURE)
+except requests.exceptions.HTTPError as errh:
+ print("\nHttp Error:", errh)
+ sys.exit(HTTP_FAILURE)
+except requests.exceptions.ConnectionError as errc:
+ print("\nError Connecting:", errc)
+ sys.exit(HTTP_FAILURE)
+except requests.exceptions.Timeout as errt:
+ print("\nTimeout Error:", errt)
+ sys.exit(HTTP_FAILURE)
+
+################################################################################
+
+def lambda_handler(event, context):
+ ret = dict()
+ res = []
+
+ try:
+ # The list of rows to return.
+
+ result = None
+
+ rows = event["arguments"]
+ print("number of rows")
+ print(len(rows))
+ u = p11url + "/rest/v2.0/tokenize"
+ # For each input row
+ for row in rows:
+ # Include the row number.
+ #row_number = row[0]
+ row_value = row[0]
+ #print(row_value)
+ data = {"tokengroup" : p11tokgroup, "tokentemplate" : p11toktemplate}
+ data["data"] = row_value
+ # Post the request
+ r = requests.post(u, headers=hdrs, auth=p11auth, verify=False, json=data)
+ result = json.loads(r.text)
+ encvalue = result['token']
+ res.append(encvalue)
+ ret['success'] = True
+ ret["num_records"] = len(rows)
+ ret['results'] = res
+ #json_compatible_string_to_return = json.dumps( return_value )
+ return json.dumps(ret)
+
+ except:
+ ret['success'] = False
+ ret["error_msg"] = "my function isn't working"
+ ret["num_records"] = len(rows)
+ ret['results'] = event["arguments"]
+ return json.dumps(ret)
\ No newline at end of file
diff --git a/database/redshift/src/test/java/CMUserSetHelper.java b/database/redshift/src/test/java/CMUserSetHelper.java
index fa875b4b..06f17015 100644
--- a/database/redshift/src/test/java/CMUserSetHelper.java
+++ b/database/redshift/src/test/java/CMUserSetHelper.java
@@ -1,3 +1,4 @@
+package com.example;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
@@ -59,9 +60,6 @@ public CMUserSetHelper(String usersetid, String cmIP) {
this.usersetid = usersetid;
this.cmIP = cmIP;
this.apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?name=";
- // this.apiUrlGetUsers = "https://" + cmIP +
- // "/api/v1/data-protection/user-sets/" + usersetid + "/users?limit=" +
- // this.usersetlimit + "&name=";
this.addusertouserset = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users";
this.authUrl = "https://" + cmIP + "/api/v1/auth/tokens";
}
@@ -73,12 +71,9 @@ public static void main(String[] args) throws Exception {
String cmip = args[2];
String usersetid = args[3];
String filePath = args[4];
- // CMUserSetHelper("716f01a6-5cab-4799-925a-6dc2d8712fc1","20.241.70.238");
- // CMUserSetHelper("32d89a8d-efac-4c50-9b53-f51d0c03413e",
CMUserSetHelper cmusersetHelper = new CMUserSetHelper(usersetid, cmip);
int totalrecords = 0;
- // String apiUrl = apiUrlGetUsers;
String jwthtoken = geAuthToken(cmusersetHelper.authUrl, username, password);
String newtoken = "Bearer " + removeQuotes(jwthtoken);
@@ -92,6 +87,7 @@ public static void main(String[] args) throws Exception {
if (cmusersetHelper.chunksize > totoalnbrofrecords) {
cmusersetHelper.chunksize = totoalnbrofrecords / 2;
}
+ //totalrecords = cmusersetHelper.addAUserToUserSet(cmusersetHelper.addusertouserset, newtoken);
totalrecords = cmusersetHelper.addAUserToUserSetFromFile(cmusersetHelper.addusertouserset, newtoken, filePath);
System.out.println("Totalrecords inserted into Userset " + cmusersetHelper.usersetid + " = " + totalrecords);
@@ -186,7 +182,6 @@ public int addAUserToUserSetFromFile(String url, String newtoken, String filePat
while ((line = br.readLine()) != null) {
totalnbrofrecords++;
- // Append the email address to the payload
payloadBuilder.append("\"").append(line).append("\",");
count++;
@@ -371,7 +366,7 @@ public static String geAuthToken(String apiUrl, String usernb, String pwd) throw
String jStr = "{\"username\":\"" + usernb + "\",\"password\":\"" + pwd + "\"}";
disableCertValidation();
- String totalstr = null;
+ String jwtstr = null;
try {
URL url = new URL(apiUrl);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
@@ -392,22 +387,22 @@ public static String geAuthToken(String apiUrl, String usernb, String pwd) throw
reader.close();
JsonObject input = null;
- JsonElement total = null;
+ JsonElement jwt = null;
JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject();
if (rootNode.isJsonObject()) {
input = rootNode.getAsJsonObject();
if (input.isJsonObject()) {
- total = input.get("jwt");
+ jwt = input.get("jwt");
}
}
- JsonPrimitive column = total.getAsJsonPrimitive();
- totalstr = column.getAsJsonPrimitive().toString();
+ JsonPrimitive column = jwt.getAsJsonPrimitive();
+ jwtstr = column.getAsJsonPrimitive().toString();
connection.disconnect();
} catch (Exception e) {
e.printStackTrace();
}
- return totalstr;
+ return jwtstr;
}
@@ -431,7 +426,7 @@ public static String geAuthToken(String apiUrl) throws Exception
disableCertValidation();
- String totalstr = null;
+ String jwtstr = null;
try {
URL url = new URL(apiUrl);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
@@ -453,22 +448,22 @@ public static String geAuthToken(String apiUrl) throws Exception
if (debug)
System.out.println("response " + response);
JsonObject input = null;
- JsonElement total = null;
+ JsonElement jwt = null;
JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject();
if (rootNode.isJsonObject()) {
input = rootNode.getAsJsonObject();
if (input.isJsonObject()) {
- total = input.get("jwt");
+ jwt = input.get("jwt");
}
}
- JsonPrimitive column = total.getAsJsonPrimitive();
- totalstr = column.getAsJsonPrimitive().toString();
+ JsonPrimitive column = jwt.getAsJsonPrimitive();
+ jwtstr = column.getAsJsonPrimitive().toString();
connection.disconnect();
} catch (Exception e) {
e.printStackTrace();
}
- return totalstr;
+ return jwtstr;
}
diff --git a/database/redshift/src/test/java/ThalesAWSRedshiftCADPBulkTester.java b/database/redshift/src/test/java/ThalesAWSRedshiftCADPBulkTester.java
index cbd313fb..77322869 100644
--- a/database/redshift/src/test/java/ThalesAWSRedshiftCADPBulkTester.java
+++ b/database/redshift/src/test/java/ThalesAWSRedshiftCADPBulkTester.java
@@ -32,7 +32,7 @@
* to protect sensitive data in a column. This example uses
* Format Preserve Encryption (FPE) to maintain the original format of the data
* so applications or business intelligence tools do not have to change in order
- * to use these columns. There is no need to deploy a function to run it.
+ * to use these columns.
*
* Note: This source code is only to be used for testing and proof of concepts.
* Not production ready code. Was not tested for all possible data sizes and
@@ -57,9 +57,11 @@ public class ThalesAWSRedshiftCADPBulkTester implements RequestStreamHandler {
private static final Logger logger = Logger.getLogger(ThalesAWSRedshiftCADPBulkTester.class.getName());
private static final Gson gson = new Gson();
+ private static final String BADDATATAG = new String("9999999999999999");
+
private static byte[][] data;
private static AlgorithmParameterSpec[] spec;
- private static int BATCHLIMIT = 10000;
+ private static int BATCHLIMIT = 2;
public static void main(String[] args) throws Exception {
@@ -72,7 +74,15 @@ public static void main(String[] args) throws Exception {
+ " [ \"thisisthefirstvalue\"],\r\n" + " [ \"Thisisthesecondvalue\"],\r\n"
+ " [ \"Thisisthethirdvalue\"]\r\n" + " ]\r\n" + " }";
- nw2.handleRequest(request, null, null);
+ String request_nbr = "{\r\n" + " \"request_id\" : \"23FF1F97-F28A-44AA-AB67-266ED976BF40\",\r\n"
+ + " \"cluster\" : \"arn:aws:redshift:xxxx\",\r\n" + " \"user\" : \"adminuser\",\r\n"
+ + " \"database\" : \"db1\",\r\n" + " \"external_function\": \"public.foo\",\r\n"
+ + " \"query_id\" : 5678234,\r\n" + " \"num_records\" : 7,\r\n" + " \"arguments\" : [\r\n"
+ + " [ \"5678234\"],\r\n" + " [ \"434534535\"],\r\n"
+ + " [ \"56446\"],\r\n" + " [ \"4\"],\r\n" + " [ \"\"],\r\n" + " [ \"56\"],\r\n"
+ + " [ \"null\"]\r\n" + " ]\r\n" + " }";
+
+ nw2.handleRequest(request_nbr, null, null);
}
@@ -106,9 +116,6 @@ public void handleRequest(String inputStream, OutputStream outputStream, Context
Map encryptedErrorMap = new HashMap();
// https://www.baeldung.com/java-aws-lambda
- StringBuffer redshiftreturndatasb = new StringBuffer();
- StringBuffer redshiftreturndatasc = new StringBuffer();
-
NAESession session = null;
if (rootNode.isJsonObject()) {
@@ -139,95 +146,87 @@ public void handleRequest(String inputStream, OutputStream outputStream, Context
String userName = System.getenv("CMUSER");
String password = System.getenv("CMPWD");
- // returnciphertextforuserwithnokeyaccess = is a environment variable to express
- // how data should be
- // returned when the user above does not have access to the key and if doing a
- // lookup in the userset
- // and the user does not exist. If returnciphertextforuserwithnokeyaccess = null
- // then an error will be
- // returned to the query, else the results set will provide ciphertext.
- // validvalues are 1 or null
- // 1 will return cipher text
- // null will return error.
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned
+ // when the user above does not have access to the key and if doing a
+ // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no
+ // then an error will be returned to the query, else the results set will provide ciphertext.
String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
- boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.matches("-?\\d+"); // Using regular
-
- // usersetlookup = should a userset lookup be done on the user from Big Query? 1
- // = true 0 = false.
+ // yes,no
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB
+ // yes,no
String usersetlookup = System.getenv("usersetlookup");
- // usersetID = should be the usersetid in CM to query.
+ // usersetidincm = should be the usersetid in CM to query.
String usersetID = System.getenv("usersetidincm");
- // usersetlookupip = this is the IP address to query the userset. Currently it
- // is
- // the userset in CM but could be a memcache or other in memory db.
+ // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be
+ // a memcache or other in memory db.
String userSetLookupIP = System.getenv("usersetlookupip");
- boolean usersetlookupbool = usersetlookup.matches("-?\\d+");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
int batchsize = Integer.parseInt(System.getenv("BATCHSIZE"));
+ int totalNbrofRows = redshiftdata.size();
+ int totalRowsLeft = totalNbrofRows;
+ String mode = System.getenv("mode");
+ String datatype = System.getenv("datatype");
+
+
+ if (batchsize > totalNbrofRows)
+ batchsize = totalNbrofRows;
if (batchsize >= BATCHLIMIT)
batchsize = BATCHLIMIT;
+
+
spec = new FPEParameterAndFormatSpec[batchsize];
data = new byte[batchsize][];
-
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+
try {
- System.setProperty("com.ingrian.security.nae.IngrianNAE_Properties_Conf_Filename",
- "D:\\product\\Build\\IngrianNAE-134.properties");
-
- /*
- * System.setProperty(
- * "com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
- * "CADP_for_JAVA.properties"); IngrianProvider builder = new
- * Builder().addConfigFileInputStream(
- * getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).
- * build();
- */
+ System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
+ "D:\\product\\Build\\CADP_for_JAVA.properties");
+
session = NAESession.getSession(userName, password.toCharArray());
NAEKey key = NAEKey.getSecretKey(keyName, session);
int row_number = 0;
-
- // Serialization
- redshiftreturndatasb.append("{ \"success\":");
- redshiftreturndatasb.append(status);
- redshiftreturndatasb.append(",");
- redshiftreturndatasb.append(" \"num_records\":");
- redshiftreturndatasb.append(nbr_of_rows_json_int);
- redshiftreturndatasb.append(",");
- redshiftreturndatasb.append(" \"results\": [");
-
+
if (usersetlookupbool) {
- // Convert the string to an integer
- int num = Integer.parseInt(usersetlookup);
- // make sure cmuser is in Application Data Protection Clients Group
- if (num >= 1) {
- boolean founduserinuserset = true;
- try {
- founduserinuserset = findUserInUserSet(redshiftuserstr, userName, password, usersetID,
- userSetLookupIP);
- } catch (Exception e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- // System.out.println("Found User " + founduserinuserset);
- if (!founduserinuserset)
- throw new CustomException("1001, User Not in User Set", 1001);
+ // make sure cmuser is in Application Data Protection Clients Group
- }
+ boolean founduserinuserset = findUserInUserSet(redshiftuserstr, userName, password, usersetID,
+ userSetLookupIP);
+ // System.out.println("Found User " + founduserinuserset);
+ if (!founduserinuserset)
+ throw new CustomException("1001, User Not in User Set", 1001);
- else
- usersetlookupbool = false;
} else {
usersetlookupbool = false;
}
- String algorithm = "FPE/FF1/CARD62";
+
+ int cipherType = 0;
+ String algorithm = "FPE/FF1v2/CARD62";
+
+ if (mode.equals("encrypt"))
+ cipherType = Cipher.ENCRYPT_MODE;
+ else
+ cipherType = Cipher.DECRYPT_MODE;
+
+ if (datatype.equals("char"))
+ algorithm = "FPE/FF1/CARD62";
+ else if (datatype.equals("charint"))
+ algorithm = "FPE/FF1/CARD10";
+ else
+ algorithm = "FPE/FF1/CARD10";
String tweakAlgo = null;
String tweakData = null;
- int totalRowsLeft = redshiftdata.size();
- AbstractNAECipher encryptCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider");
+ FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
+ .build();
- encryptCipher.init(Cipher.ENCRYPT_MODE, key, spec[0]);
+ AbstractNAECipher thalesCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider");
+
+ thalesCipher.init(cipherType, key, spec[0]);
int i = 0;
int index = 0;
@@ -236,7 +235,7 @@ public void handleRequest(String inputStream, OutputStream outputStream, Context
int dataIndex = 0;
int specIndex = 0;
- while (i < redshiftdata.size()) {
+ while (i < totalNbrofRows) {
index = 0;
if (newchunk) {
@@ -254,11 +253,31 @@ public void handleRequest(String inputStream, OutputStream outputStream, Context
JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
- JsonPrimitive redshiftcolumn = redshiftrow.get(0).getAsJsonPrimitive();
- //System.out.print(redshiftcolumn + " ");
- String sensitive = redshiftcolumn.getAsJsonPrimitive().toString();
- data[dataIndex++] = sensitive.getBytes();
- spec[specIndex++] = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo).build();
+ // insert new....
+ String sensitive = checkValid(redshiftrow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ sensitive = BADDATATAG+sensitive;
+ data[dataIndex++] = sensitive.getBytes();
+ // Can not return number since a leading 0 will not work.
+ // innerDataArray.add(new BigInteger(sensitive));
+ } else
+ data[dataIndex++] = BADDATATAG.getBytes();
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+ data[dataIndex++] = sensitive.getBytes();
+ } else {
+ data[dataIndex++] = sensitive.getBytes();
+ }
+ spec[specIndex++] = param;
+
+ System.out.println("not valid or null");
+ } else {
+ data[dataIndex++] = sensitive.getBytes();
+ spec[specIndex++] = param;
+ }
if (count == batchsize - 1) {
@@ -266,7 +285,7 @@ public void handleRequest(String inputStream, OutputStream outputStream, Context
encryptedErrorMap = new HashMap();
// performing bulk operation
- byte[][] encryptedData = encryptCipher.doFinalBulk(data, spec, encryptedErrorMap);
+ byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap);
for (Map.Entry entry : encryptedErrorMap.entrySet()) {
Integer mkey = entry.getKey();
@@ -275,10 +294,7 @@ public void handleRequest(String inputStream, OutputStream outputStream, Context
}
for (int enc = 0; enc < encryptedData.length; enc++) {
-
- redshiftreturndatasc.append("[");
- redshiftreturndatasc.append(new String(encryptedData[enc]));
- redshiftreturndatasc.append("],");
+ dataArray.add(new String(encryptedData[enc]));
index++;
}
@@ -297,23 +313,21 @@ public void handleRequest(String inputStream, OutputStream outputStream, Context
if (count > 0) {
numberofchunks++;
- byte[][] encryptedData = encryptCipher.doFinalBulk(data, spec, encryptedErrorMap);
+ byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap);
for (int enc = 0; enc < encryptedData.length; enc++) {
-
- redshiftreturndatasc.append("[");
- redshiftreturndatasc.append(new String(encryptedData[enc]));
- redshiftreturndatasc.append("],");
+ dataArray.add(new String(encryptedData[enc]));
index++;
totalRowsLeft--;
}
}
- redshiftreturndatasc.append("] }");
- redshiftreturndatasb.append(redshiftreturndatasc);
- redshiftreturndatasb.append("}");
-
- redshiftreturnstring = new String(redshiftreturndatasb);
+
+ bodyObject.addProperty("success", true);
+ bodyObject.addProperty("num_records", totalNbrofRows);
+ bodyObject.add("results", dataArray);
+ redshiftreturnstring = bodyObject.toString();
+
} catch (
@@ -328,16 +342,14 @@ public void handleRequest(String inputStream, OutputStream outputStream, Context
JsonPrimitive redshiftcolumn = redshiftrow.get(0).getAsJsonPrimitive();
String sensitive = redshiftcolumn.getAsJsonPrimitive().toString();
-
+ dataArray.add(sensitive);
redshiftreturndata.append(sensitive);
- if (redshiftdata.size() == 1 || i == redshiftdata.size() - 1)
- continue;
- else
- redshiftreturndata.append(",");
- }
- redshiftreturndata.append("]}");
- redshiftreturnstring = new String(redshiftreturndata);
+ }
+ bodyObject.addProperty("success", true);
+ bodyObject.addProperty("num_records", totalNbrofRows);
+ bodyObject.add("results", dataArray);
+ redshiftreturnstring = bodyObject.toString();
} else {
statusCode = 400;
@@ -357,12 +369,7 @@ public void handleRequest(String inputStream, OutputStream outputStream, Context
session.closeSession();
}
}
- int lastIndex = redshiftreturnstring.lastIndexOf(",");
- // Replace the comma before the closing square bracket if it exists
- if (lastIndex != -1) {
- redshiftreturnstring = redshiftreturnstring.substring(0, lastIndex)
- + redshiftreturnstring.substring(lastIndex + 1);
- }
+
System.out.println("string = " + redshiftreturnstring);
System.out.println("numberofchunks = " + numberofchunks);
// outputStream.write(new Gson().toJson(redshiftreturnstring).getBytes());
@@ -374,6 +381,29 @@ public void handleRequest(InputStream input, OutputStream output, Context contex
}
+ public String checkValid(JsonArray redshiftrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (redshiftrow != null && redshiftrow.size() > 0) {
+ JsonElement element = redshiftrow.get(0);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
String userSetLookupIP) throws Exception {
diff --git a/database/redshift/src/test/java/ThalesAWSRedshiftCADPTester.java b/database/redshift/src/test/java/ThalesAWSRedshiftCADPTester.java
index c9966dfe..0d5f88f6 100644
--- a/database/redshift/src/test/java/ThalesAWSRedshiftCADPTester.java
+++ b/database/redshift/src/test/java/ThalesAWSRedshiftCADPTester.java
@@ -5,7 +5,10 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.util.logging.Logger;
+
+import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
import javax.crypto.spec.IvParameterSpec;
import org.apache.commons.io.IOUtils;
import com.google.gson.Gson;
@@ -43,15 +46,15 @@
public class ThalesAWSRedshiftCADPTester implements RequestStreamHandler {
private static final Logger logger = Logger.getLogger(ThalesAWSRedshiftCADPTester.class.getName());
private static final Gson gson = new Gson();
+ private static final String BADDATATAG = new String("9999999999999999");
/**
* Returns an String that will be the encrypted value
*
- * Examples: select thales_token_cadp_char(eventname) as enceventname ,
- * eventname from event where len(eventname) > 5
+ * Examples: select thales_token_cadp_char(eventname) as enceventname , eventname from event where len(eventname) >
+ * 5
*
- * @param is any column in the database or any value that needs to be encrypted.
- * Mostly used for ELT processes.
+ * @param is any column in the database or any value that needs to be encrypted. Mostly used for ELT processes.
*/
public static void main(String[] args) throws Exception {
@@ -61,30 +64,37 @@ public static void main(String[] args) throws Exception {
String request = "{\r\n" + " \"request_id\" : \"23FF1F97-F28A-44AA-AB67-266ED976BF40\",\r\n"
+ " \"cluster\" : \"arn:aws:redshift:xxxx\",\r\n" + " \"user\" : \"adminuser\",\r\n"
+ " \"database\" : \"db1\",\r\n" + " \"external_function\": \"public.foo\",\r\n"
- + " \"query_id\" : 5678234,\r\n" + " \"num_records\" : 3,\r\n" + " \"arguments\" : [\r\n"
+ + " \"query_id\" : 5678234,\r\n" + " \"num_records\" : 7,\r\n" + " \"arguments\" : [\r\n"
+ " [ \"thisisthefirstvalue\"],\r\n" + " [ \"Thisisthesecondvalue\"],\r\n"
+ + " [ \"null\"],\r\n" + " [ \"T\"],\r\n" + " [ \"\"],\r\n" + " [ \"A\"],\r\n"
+ " [ \"Thisisthethirdvalue\"]\r\n" + " ]\r\n" + " }";
+
+ String request_nbr = "{\r\n" + " \"request_id\" : \"23FF1F97-F28A-44AA-AB67-266ED976BF40\",\r\n"
+ + " \"cluster\" : \"arn:aws:redshift:xxxx\",\r\n" + " \"user\" : \"adminuser\",\r\n"
+ + " \"database\" : \"db1\",\r\n" + " \"external_function\": \"public.foo\",\r\n"
+ + " \"query_id\" : 5678234,\r\n" + " \"num_records\" : 7,\r\n" + " \"arguments\" : [\r\n"
+ + " [ \"5678234\"],\r\n" + " [ \"434534535\"],\r\n"
+ + " [ \"56446\"],\r\n" + " [ \"4\"],\r\n" + " [ \"\"],\r\n" + " [ \"56\"],\r\n"
+ + " [ \"null\"]\r\n" + " ]\r\n" + " }";
+
String response = null;
String testJson = null;
- nw2.handleRequest(request, null, null);
+ nw2.handleRequest(request_nbr, null, null);
}
public void handleRequest(String inputStream, String outputStream, Context context)
- throws IOException, CustomException {
+ throws IOException, CustomException, IllegalBlockSizeException, BadPaddingException {
// context.getLogger().log("Input: " + inputStream);
// String input = IOUtils.toString(inputStream, "UTF-8");
String input = inputStream;
JsonParser parser = new JsonParser();
NAESession session = null;
int statusCode = 200;
-
+
String redshiftreturnstring = null;
- StringBuffer redshiftreturndata = new StringBuffer();
- boolean status = true;
-
JsonObject redshiftinput = null;
JsonElement rootNode = parser.parse(input);
@@ -107,156 +117,99 @@ public void handleRequest(String inputStream, String outputStream, Context conte
System.out.println("Bad data from Redshift.");
}
-
+
JsonPrimitive nbr_of_rows_json = redshiftinput.getAsJsonPrimitive("num_records");
String nbr_of_rows_json_str = nbr_of_rows_json.getAsJsonPrimitive().toString();
int nbr_of_rows_json_int = new Integer(nbr_of_rows_json_str);
System.out.println("number of records " + nbr_of_rows_json_str);
-
+
String keyName = "testfaas";
String userName = System.getenv("CMUSER");
String password = System.getenv("CMPWD");
- // returnciphertextforuserwithnokeyaccess = is a environment variable to express
- // how data should be
- // returned when the user above does not have access to the key and if doing a
- // lookup in the userset
- // and the user does not exist. If returnciphertextforuserwithnokeyaccess = null
- // then an error will be
- // returned to the query, else the results set will provide ciphertext.
- // validvalues are 1 or null
- // 1 will return cipher text
- // null will return error.
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned
+ // when the user above does not have access to the key and if doing a
+ // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no
+ // then an error will be returned to the query, else the results set will provide ciphertext.
String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
- boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.matches("-?\\d+"); // Using regular
-
- // usersetlookup = should a userset lookup be done on the user from Big Query? 1
- // = true 0 = false.
+ // yes,no
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB
+ // yes,no
String usersetlookup = System.getenv("usersetlookup");
- // usersetID = should be the usersetid in CM to query.
+ // usersetidincm = should be the usersetid in CM to query.
String usersetID = System.getenv("usersetidincm");
- // usersetlookupip = this is the IP address to query the userset. Currently it
- // is
- // the userset in CM but could be a memcache or other in memory db.
+ // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be
+ // a memcache or other in memory db.
String userSetLookupIP = System.getenv("usersetlookupip");
- boolean usersetlookupbool = usersetlookup.matches("-?\\d+");
-
-
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ String mode = System.getenv("mode");
+ String datatype = System.getenv("datatype");
try {
-
- // Serialization
- redshiftreturndata.append("{ \"success\":");
- redshiftreturndata.append(status);
- redshiftreturndata.append(",");
- redshiftreturndata.append(" \"num_records\":");
- redshiftreturndata.append(nbr_of_rows_json_int);
- redshiftreturndata.append(",");
- redshiftreturndata.append(" \"results\": [");
-
-
-
if (usersetlookupbool) {
- // Convert the string to an integer
- int num = Integer.parseInt(usersetlookup);
// make sure cmuser is in Application Data Protection Clients Group
- if (num >= 1) {
- boolean founduserinuserset = true;
- try {
- founduserinuserset = findUserInUserSet(redshiftuserstr, userName, password, usersetID,
- userSetLookupIP);
- } catch (Exception e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- // System.out.println("Found User " + founduserinuserset);
- if (!founduserinuserset)
- throw new CustomException("1001, User Not in User Set", 1001);
- }
+ boolean founduserinuserset = findUserInUserSet(redshiftuserstr, userName, password, usersetID,
+ userSetLookupIP);
+ // System.out.println("Found User " + founduserinuserset);
+ if (!founduserinuserset)
+ throw new CustomException("1001, User Not in User Set", 1001);
- else
- usersetlookupbool = false;
} else {
usersetlookupbool = false;
}
-
-
- System.setProperty("com.ingrian.security.nae.IngrianNAE_Properties_Conf_Filename",
- "D:\\product\\Build\\IngrianNAE-134.properties");
-
- // System.setProperty("com.ingrian.security.nae.NAE_IP.1", "10.20.1.9");
// System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
// "CADP_for_JAVA.properties");
+ System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
+ "D:\\product\\Build\\CADP_for_JAVA.properties");
// IngrianProvider builder = new Builder().addConfigFileInputStream(
// getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
session = NAESession.getSession(userName, password.toCharArray());
NAEKey key = NAEKey.getSecretKey(keyName, session);
+ int cipherType = 0;
+ String algorithm = "FPE/FF1/CARD62";
+ if (mode.equals("encrypt"))
+ cipherType = Cipher.ENCRYPT_MODE;
+ else
+ cipherType = Cipher.DECRYPT_MODE;
+
+ if (datatype.equals("char"))
+ algorithm = "FPE/FF1/CARD62";
+ else if (datatype.equals("charint"))
+ algorithm = "FPE/FF1/CARD10";
+ else
+ algorithm = "FPE/FF1/CARD10";
- String algorithm = "FPE/FF1/CARD62";
- // String algorithm = "AES/CBC/PKCS5Padding";
String tweakAlgo = null;
String tweakData = null;
FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
.build();
- Cipher encryptCipher = Cipher.getInstance(algorithm, "IngrianProvider");
- String encdata = "";
-
- for (int i = 0; i < redshiftdata.size(); i++) {
- JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
-
- JsonPrimitive redshiftcolumn = redshiftrow.get(0).getAsJsonPrimitive();
-
- String sensitive = redshiftcolumn.getAsJsonPrimitive().toString();
-
- // initialize cipher to encrypt.
- encryptCipher.init(Cipher.ENCRYPT_MODE, key, param);
- // encrypt data
- byte[] outbuf = encryptCipher.doFinal(sensitive.getBytes());
- encdata = new String(outbuf);
- System.out.println("Enc data : " + encdata);
-
- redshiftreturndata.append(encdata);
- if (redshiftdata.size() == 1 || i == redshiftdata.size() - 1)
- continue;
- else
- redshiftreturndata.append(",");
- }
+ Cipher thalesCipher = Cipher.getInstance(algorithm, "IngrianProvider");
- redshiftreturndata.append("]}");
+ // initialize cipher to encrypt.
+ thalesCipher.init(cipherType, key, param);
- redshiftreturnstring = new String(redshiftreturndata);
- System.out.println("string = " + redshiftreturnstring);
+ redshiftreturnstring = formatReturnValue(200, redshiftdata, false, thalesCipher, datatype);
+ System.out.println("orig string = " + redshiftreturnstring);
// System.out.println(new Gson().toJson(redshiftreturnstring));
} catch (Exception e) {
+
System.out.println("In Exception with = " + e.getMessage());
if (returnciphertextbool) {
- if (e.getMessage().contains("1401") || (e.getMessage().contains("1001") || (e.getMessage().contains("1002"))) ) {
- for (int i = 0; i < redshiftdata.size(); i++) {
- JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
-
- JsonPrimitive redshiftcolumn = redshiftrow.get(0).getAsJsonPrimitive();
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
- String sensitive = redshiftcolumn.getAsJsonPrimitive().toString();
-
- redshiftreturndata.append(sensitive);
- if (redshiftdata.size() == 1 || i == redshiftdata.size() - 1)
- continue;
- else
- redshiftreturndata.append(",");
- }
- redshiftreturndata.append("]}");
+ redshiftreturnstring = formatReturnValue(200, redshiftdata, true, null, datatype);
- redshiftreturnstring = new String(redshiftreturndata);
-
} else {
statusCode = 400;
redshiftreturnstring = formatReturnValue(statusCode);
@@ -299,6 +252,99 @@ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd,
}
+ public String checkValid(JsonArray redShiftRow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (redShiftRow != null && redShiftRow.size() > 0) {
+ JsonElement element = redShiftRow.get(0);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ public String formatReturnValue(int statusCode, JsonArray redShiftArray, boolean error, Cipher thalesCipher,
+ String datatype) throws IllegalBlockSizeException, BadPaddingException {
+ int row_number = 0;
+
+ String encdata = null;
+ String sensitive = null;
+ String formattedString = null;
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+ int nbrofrecords = redShiftArray.size();
+
+ for (int i = 0; i < nbrofrecords; i++) {
+ JsonArray redshiftRow = redShiftArray.get(i).getAsJsonArray();
+
+ sensitive = checkValid(redshiftRow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ System.out.println("adding null not charint or nbr");
+ sensitive = sensitive.replace("notvalid", "");
+ dataArray.add(sensitive);
+ // Can not return number since a leading 0 will not work.
+ // innerDataArray.add(new BigInteger(sensitive));
+ } else
+ dataArray.add(BADDATATAG);
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+ dataArray.add("");
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ dataArray.add(sensitive);
+ } else {
+ dataArray.add(sensitive);
+ }
+
+ } else {
+ if (!error) {
+ byte[] outbuf = thalesCipher.doFinal(sensitive.getBytes());
+ encdata = new String(outbuf);
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ dataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ dataArray.add(encdata);
+ }
+ } else {
+ encdata = sensitive;
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ dataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ dataArray.add(encdata);
+ }
+ }
+ }
+
+ // innerDataArray.add(encdata);
+
+ }
+ bodyObject.addProperty("success", true);
+ bodyObject.addProperty("num_records", nbrofrecords);
+ bodyObject.add("results", dataArray);
+ formattedString = bodyObject.toString();
+ return formattedString;
+ // return bodyString;
+
+ }
+
+
public String formatReturnValue(int statusCode)
{
diff --git a/database/redshift/src/test/java/ThalesAWSRedshiftCRDPBulkTester.java b/database/redshift/src/test/java/ThalesAWSRedshiftCRDPBulkTester.java
new file mode 100644
index 00000000..992af460
--- /dev/null
+++ b/database/redshift/src/test/java/ThalesAWSRedshiftCRDPBulkTester.java
@@ -0,0 +1,579 @@
+
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.io.IOUtils;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+
+import okhttp3.MediaType;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+
+/*
+ * This test app to test the logic for a AWS Redshift Database User Defined
+ * Function(UDF). It is an example of how to use Thales Cipher REST Data Protection (CRDP)
+ * to protect sensitive data in a column. This example uses
+ * Format Preserve Encryption (FPE) to maintain the original format of the data
+ * so applications or business intelligence tools do not have to change in order
+ * to use these columns. There is no need to deploy a function to run it.
+ *
+ * This uses the protectbulk and revealbulk api's.
+ * Note: This source code is only to be used for testing and proof of concepts.
+ * Not production ready code. Was tested with CM 2.14 &
+ * CRDP 1.0 tech preview For more information on CRDP see link below.
+ * https://thalesdocs.com/ctp/con/crdp/latest/admin/index.html
+ *
+ * @author mwarner
+ *
+ */
+
+public class ThalesAWSRedshiftCRDPBulkTester implements RequestStreamHandler {
+
+ private static final Gson gson = new Gson();
+
+ private static final String BADDATATAG = new String("9999999999999999");
+ private static int BATCHLIMIT = 10000;
+
+ private static final String REVEALRETURNTAG = new String("data");
+ private static final String PROTECTRETURNTAG = new String("protected_data");
+
+ public static void main(String[] args) throws Exception {
+
+ ThalesAWSRedshiftCRDPBulkTester nw2 = new ThalesAWSRedshiftCRDPBulkTester();
+
+ String protectrequest = "{\r\n" + " \"request_id\" : \"23FF1F97-F28A-44AA-AB67-266ED976BF40\",\r\n"
+ + " \"cluster\" : \"arn:aws:redshift:xxxx\",\r\n" + " \"user\" : \"adminuser\",\r\n"
+ + " \"database\" : \"db1\",\r\n" + " \"external_function\": \"public.foo\",\r\n"
+ + " \"query_id\" : 5678234,\r\n" + " \"num_records\" : 5,\r\n" + " \"arguments\" : [\r\n"
+ + " [ \"5678234\"],\r\n" + " [ \"45345366345\"],\r\n" + " [ \"2342342342424\"],\r\n"
+ + " [ \"2424\"],\r\n" + " [ \"234234234255667777\"]\r\n" + " ]\r\n" + " }";
+
+ String protectrequestnull = "{\r\n" + " \"request_id\" : \"23FF1F97-F28A-44AA-AB67-266ED976BF40\",\r\n"
+ + " \"cluster\" : \"arn:aws:redshift:xxxx\",\r\n" + " \"user\" : \"adminuser\",\r\n"
+ + " \"database\" : \"db1\",\r\n" + " \"external_function\": \"public.foo\",\r\n"
+ + " \"query_id\" : 5678234,\r\n" + " \"num_records\" : 5,\r\n" + " \"arguments\" : [\r\n"
+ + " [ \"5678234\"],\r\n" + " [ \"45345366345\"],\r\n" + " [ \"2342342342424\"],\r\n"
+ + " [ \"\"],\r\n" + " [ \"null\"]\r\n" + " ]\r\n" + " }";
+
+ String revealrequest = "{\r\n" + " \"request_id\" : \"23FF1F97-F28A-44AA-AB67-266ED976BF40\",\r\n"
+ + " \"cluster\" : \"arn:aws:redshift:xxxx\",\r\n" + " \"user\" : \"admin\",\r\n"
+ + " \"database\" : \"db1\",\r\n" + " \"external_function\": \"public.foo\",\r\n"
+ + " \"query_id\" : 5678234,\r\n" + " \"num_records\" : 5,\r\n" + " \"arguments\" : [\r\n"
+ + " [ \"1004001jKVC6JO\"],\r\n" + " [ \"1004001DW9FaQxLPj3\"],\r\n"
+ + " [ \"1004001rJivEodgILZg8\"],\r\n" + " [ \"1004001JvcC\"],\r\n"
+ + " [ \"1004001ly4A5IY1j7QRDti4A2\"]\r\n" + " ]\r\n" + " }";
+
+ String revealrequest_ext = "{\r\n" + " \"request_id\" : \"23FF1F97-F28A-44AA-AB67-266ED976BF40\",\r\n"
+ + " \"cluster\" : \"arn:aws:redshift:xxxx\",\r\n" + " \"user\" : \"admin\",\r\n"
+ + " \"database\" : \"db1\",\r\n" + " \"external_function\": \"public.foo\",\r\n"
+ + " \"query_id\" : 5678234,\r\n" + " \"num_records\" : 5,\r\n" + " \"arguments\" : [\r\n"
+ + " [ \"LmCJFjr\"],\r\n" + " [ \"SsoB9Lalcga\"],\r\n" + " [ \"p2TryIkXAjCLk\"],\r\n"
+ + " [ \"J5IX\"],\r\n" + " [ \"cWENACcmC12zWDxNpq\"]\r\n" + " ]\r\n" + " }";
+
+ nw2.handleRequest(revealrequest, null, null);
+
+ }
+
+ /**
+ * Returns an String that will be the encrypted value
+ *
+ */
+
+ public void handleRequest(String inputStream, OutputStream outputStream, Context context) throws IOException {
+
+ String input = inputStream;
+ JsonParser parser = new JsonParser();
+ int statusCode = 200;
+
+ String redshiftreturnstring = null;
+ StringBuffer redshiftreturndata = new StringBuffer();
+
+ int numberofchunks = 0;
+ JsonObject redshiftinput = null;
+ JsonElement rootNode = parser.parse(input);
+ JsonArray redshiftdata = null;
+ String redshiftuserstr = null;
+
+ Map reshift_ErrorMap = new HashMap();
+ // https://www.baeldung.com/java-aws-lambda
+
+ StringBuffer protection_policy_buff = new StringBuffer();
+
+ JsonObject result = new JsonObject();
+ JsonArray replies = new JsonArray();
+
+ if (rootNode.isJsonObject()) {
+ redshiftinput = rootNode.getAsJsonObject();
+ if (redshiftinput != null) {
+ redshiftdata = redshiftinput.getAsJsonArray("arguments");
+ JsonPrimitive userjson = redshiftinput.getAsJsonPrimitive("user");
+ redshiftuserstr = userjson.getAsJsonPrimitive().toString();
+ redshiftuserstr = redshiftuserstr.replace("\"", "");
+ } else {
+ System.out.println("Root node not found.");
+ }
+ } else {
+ System.out.println("Bad data from snowflake.");
+ }
+
+ JsonPrimitive nbr_of_rows_json = redshiftinput.getAsJsonPrimitive("num_records");
+ String nbr_of_rows_json_str = nbr_of_rows_json.getAsJsonPrimitive().toString();
+ int nbr_of_rows_json_int = Integer.parseInt(nbr_of_rows_json_str);
+
+ String crdpip = System.getenv("CRDPIP");
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned
+ // when the user above does not have access to the key and if doing a
+ // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no
+ // then an error will be returned to the query, else the results set will provide ciphertext.
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ // yes,no
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB
+ // yes,no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be
+ // a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ String keymetadatalocation = System.getenv("keymetadatalocation");
+ String external_version_from_ext_source = System.getenv("keymetadata");
+ String protection_profile = System.getenv("protection_profile");
+ String mode = System.getenv("mode");
+ String datatype = System.getenv("datatype");
+
+ String inputDataKey = null;
+ String outputDataKey = null;
+ String protectedData = null;
+ String externalkeymetadata = null;
+ boolean bad_data = false;
+ String notvalid = "notvalid";
+ String jsonBody = null;
+ String jsonTagForProtectReveal = null;
+
+ int error_count = 0;
+
+ // String external_version_from_ext_source = "1004001";
+ // String external_version_from_ext_source = "1001001";
+
+ JsonObject crdp_payload = new JsonObject();
+
+ String showrevealkey = "yes";
+
+ if (mode.equals("protectbulk")) {
+ inputDataKey = "data_array";
+ outputDataKey = "protected_data_array";
+ jsonTagForProtectReveal = PROTECTRETURNTAG;
+ if (keymetadatalocation.equalsIgnoreCase("internal")) {
+ showrevealkey = System.getenv("showrevealinternalkey");
+ if (showrevealkey == null)
+ showrevealkey = "yes";
+ }
+ } else {
+ inputDataKey = "protected_data_array";
+ outputDataKey = "data_array";
+ jsonTagForProtectReveal = REVEALRETURNTAG;
+ }
+
+ boolean showrevealkeybool = showrevealkey.equalsIgnoreCase("yes");
+
+ int totalNbrofRows = redshiftdata.size();
+ int totalRowsLeft = totalNbrofRows;
+ // int batchsize = Integer.parseInt(System.getenv("BATCHSIZE"));
+ int batchsize = 5;
+ if (batchsize > totalNbrofRows)
+ batchsize = totalNbrofRows;
+ if (batchsize >= BATCHLIMIT)
+ batchsize = BATCHLIMIT;
+
+ try {
+
+ System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
+ "D:\\product\\Build\\CADP_for_JAVA.properties");
+
+ /*
+ * System.setProperty( "com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
+ * "CADP_for_JAVA.properties"); IngrianProvider builder = new Builder().addConfigFileInputStream(
+ * getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")). build();
+ */
+
+ int row_number = 0;
+
+ if (usersetlookupbool) {
+ // make sure cmuser is in Application Data Protection Clients Group
+
+ boolean founduserinuserset = findUserInUserSet(redshiftuserstr, userName, password, usersetID,
+ userSetLookupIP);
+ // System.out.println("Found User " + founduserinuserset);
+ if (!founduserinuserset)
+ throw new CustomException("1001, User Not in User Set", 1001);
+
+ } else {
+ usersetlookupbool = false;
+ }
+
+ int i = 0;
+ int count = 0;
+
+ JsonArray crdp_payload_array = new JsonArray();
+
+ OkHttpClient client = new OkHttpClient().newBuilder().build();
+ MediaType mediaType = MediaType.parse("application/json");
+ String urlStr = "http://" + crdpip + ":8090/v1/" + mode;
+
+ while (i < totalNbrofRows) {
+
+ String sensitive = null;
+ JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
+
+ // insert new....
+ sensitive = checkValid(redshiftrow);
+
+ protection_profile = protection_profile.trim();
+ // Format the output
+ String formattedElement = String.format("\"protection_policy_name\" : \"%s\"", protection_profile);
+ protection_policy_buff.append(formattedElement);
+ protection_policy_buff.append(",");
+
+ if (mode.equals("protectbulk")) {
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ // System.out.println("adding null not charint or nbr");
+ sensitive = sensitive.replace("notvalid", "");
+ sensitive = BADDATATAG + sensitive;
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ // sensitive = sensitive.replace("notvalid", "");
+
+ }
+
+ }
+ crdp_payload_array.add(sensitive);
+ } else {
+ JsonObject protectedDataObject = new JsonObject();
+ protectedDataObject.addProperty("protected_data", sensitive);
+ if (keymetadatalocation.equalsIgnoreCase("external")) {
+ protectedDataObject.addProperty("external_version", external_version_from_ext_source);
+ }
+ crdp_payload_array.add(protectedDataObject);
+
+ }
+ if (count == batchsize - 1) {
+
+ crdp_payload.add(inputDataKey, crdp_payload_array);
+ String inputdataarray = null;
+ if (mode.equals("revealbulk")) {
+ crdp_payload.addProperty("username", redshiftuserstr);
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ jsonBody = protection_policy_buff.toString();
+ jsonBody = jsonBody.replaceFirst("\\{", " ");
+
+ } else {
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ inputdataarray = protection_policy_buff.toString();
+ jsonBody = inputdataarray.replace("{", " ");
+ }
+ jsonBody = "{" + jsonBody;
+
+ RequestBody body = RequestBody.create(mediaType, jsonBody);
+
+ Request crdp_request = new Request.Builder()
+ // .url("http://192.168.159.143:8090/v1/protect").method("POST", body)
+ .url(urlStr).method("POST", body).addHeader("Content-Type", "application/json").build();
+ Response crdp_response = client.newCall(crdp_request).execute();
+
+ if (crdp_response.isSuccessful()) {
+ // Parse JSON response
+ String responseBody = crdp_response.body().string();
+ JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class);
+ JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey);
+
+ String status = jsonObject.get("status").getAsString();
+ int success_count = jsonObject.get("success_count").getAsInt();
+ error_count = jsonObject.get("error_count").getAsInt();
+ if (error_count > 0)
+ System.out.println("errors " + error_count);
+ for (JsonElement element : protectedDataArray) {
+
+ JsonObject protectedDataObject = element.getAsJsonObject();
+ if (protectedDataObject.has(jsonTagForProtectReveal)) {
+
+ protectedData = protectedDataObject.get(jsonTagForProtectReveal).getAsString();
+
+ if (keymetadatalocation.equalsIgnoreCase("internal")
+ && mode.equalsIgnoreCase("protectbulk") && !showrevealkeybool) {
+ if (protectedData.length() > 7)
+ protectedData = protectedData.substring(7);
+ }
+ replies.add(new String(protectedData));
+
+ if (mode.equals("protectbulk")) {
+ if (keymetadatalocation.equalsIgnoreCase("external")
+ && mode.equalsIgnoreCase("protectbulk")) {
+ externalkeymetadata = protectedDataObject.get("external_version").getAsString();
+ // System.out.println("Protected Data ext key metadata need to store this: "
+ // + externalkeymetadata);
+
+ }
+ }
+ } else if (protectedDataObject.has("error_message")) {
+ String errorMessage = protectedDataObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ reshift_ErrorMap.put(i, errorMessage);
+ bad_data = true;
+ } else
+ System.out.println("unexpected json value from results: ");
+
+ }
+
+ crdp_response.close();
+ crdp_payload_array = new JsonArray();
+ protection_policy_buff = new StringBuffer();
+ numberofchunks++;
+
+ count = 0;
+ } else {
+ System.err.println("Request failed with status code: " + crdp_response.code());
+ }
+ } else {
+ count++;
+ }
+
+ totalRowsLeft--;
+ i++;
+
+ }
+
+ if (count > 0) {
+ // if (count == batchsize - 1 || (totalRowsLeft <= totalcount)) {
+ crdp_payload.add(inputDataKey, crdp_payload_array);
+ String inputdataarray = null;
+ if (mode.equals("revealbulk")) {
+ crdp_payload.addProperty("username", redshiftuserstr);
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ jsonBody = protection_policy_buff.toString();
+ jsonBody = jsonBody.replaceFirst("\\{", " ");
+
+ } else {
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ inputdataarray = protection_policy_buff.toString();
+ jsonBody = inputdataarray.replace("{", " ");
+ }
+ jsonBody = "{" + jsonBody;
+
+ RequestBody body = RequestBody.create(mediaType, jsonBody);
+
+ Request crdp_request = new Request.Builder()
+ // .url("http://192.168.159.143:8090/v1/protect").method("POST", body)
+ .url(urlStr).method("POST", body).addHeader("Content-Type", "application/json").build();
+ Response crdp_response = client.newCall(crdp_request).execute();
+ String crdpreturnstr = null;
+ if (crdp_response.isSuccessful()) {
+ // Parse JSON response
+ String responseBody = crdp_response.body().string();
+ JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class);
+ JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey);
+
+ String status = jsonObject.get("status").getAsString();
+ int success_count = jsonObject.get("success_count").getAsInt();
+ error_count = jsonObject.get("error_count").getAsInt();
+ if (error_count > 0)
+ System.out.println("errors " + error_count);
+ for (JsonElement element : protectedDataArray) {
+
+ JsonObject protectedDataObject = element.getAsJsonObject();
+ if (protectedDataObject.has(jsonTagForProtectReveal)) {
+
+ protectedData = protectedDataObject.get(jsonTagForProtectReveal).getAsString();
+ if (keymetadatalocation.equalsIgnoreCase("internal") && mode.equalsIgnoreCase("protectbulk")
+ && !showrevealkeybool) {
+ if (protectedData.length() > 7)
+ protectedData = protectedData.substring(7);
+ }
+
+ replies.add(new String(protectedData));
+
+ if (mode.equals("protectbulk")) {
+ if (keymetadatalocation.equalsIgnoreCase("external")
+ && mode.equalsIgnoreCase("protectbulk")) {
+ externalkeymetadata = protectedDataObject.get("external_version").getAsString();
+ // System.out.println("Protected Data ext key metadata need to store this: "
+ // + externalkeymetadata);
+
+ }
+ }
+ } else if (protectedDataObject.has("error_message")) {
+ String errorMessage = protectedDataObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ reshift_ErrorMap.put(i, errorMessage);
+ bad_data = true;
+ } else
+ System.out.println("unexpected json value from results: ");
+
+ }
+
+ crdp_response.close();
+ crdp_payload_array = new JsonArray();
+ protection_policy_buff = new StringBuffer();
+ numberofchunks++;
+ // totalcount = totalcount + count;
+ count = 0;
+ } else {
+ System.err.println("Request failed with status code: " + crdp_response.code());
+ }
+ }
+ System.out.println("total chuncks " + numberofchunks);
+
+ result.addProperty("success", true);
+ result.addProperty("num_records", totalNbrofRows);
+ result.add("results", replies);
+ redshiftreturnstring = result.toString();
+
+ System.out.println("new value = " + redshiftreturnstring);
+
+ } catch (
+
+ Exception e) {
+ System.out.println("in exception with " + e.getMessage());
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
+
+ for (int i = 0; i < redshiftdata.size(); i++) {
+ JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
+
+ JsonPrimitive redshiftcolumn = redshiftrow.get(0).getAsJsonPrimitive();
+
+ String sensitive = redshiftcolumn.getAsJsonPrimitive().toString();
+ replies.add(sensitive);
+ redshiftreturndata.append(sensitive);
+
+ }
+ result.addProperty("success", true);
+ result.addProperty("num_records", totalNbrofRows);
+ result.add("results", replies);
+ redshiftreturnstring = result.toString();
+
+ } else {
+ statusCode = 400;
+ redshiftreturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+
+ } else {
+ statusCode = 400;
+ redshiftreturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ }
+
+ finally {
+
+ }
+
+ System.out.println("string = " + redshiftreturnstring);
+ System.out.println("numberofchunks = " + numberofchunks);
+
+ if (bad_data) {
+ System.out.println("errors: ");
+ for (Map.Entry entry : reshift_ErrorMap.entrySet()) {
+ Integer mkey = entry.getKey();
+ String mvalue = entry.getValue();
+ System.out.println("Error index: " + mkey);
+ System.out.println("Error Message: " + mvalue);
+ }
+
+ }
+
+ // outputStream.write(new Gson().toJson(redshiftreturnstring).getBytes());
+ }
+
+ @Override
+ public void handleRequest(InputStream input, OutputStream output, Context context) throws IOException {
+ // TODO Auto-generated method stub
+
+ }
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+
+ boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
+
+ return founduserinuserset;
+
+ }
+
+ public String checkValid(JsonArray redshiftrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (redshiftrow != null && redshiftrow.size() > 0) {
+ JsonElement element = redshiftrow.get(0);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer redshiftreturndata = new StringBuffer();
+
+ String errormsg = "\"Error in UDF \"";
+ redshiftreturndata.append("{ \"success\":");
+ redshiftreturndata.append(false);
+ redshiftreturndata.append(" \"num_records\":");
+ redshiftreturndata.append(0);
+ redshiftreturndata.append(",");
+ redshiftreturndata.append(" \"error_msg\":");
+ redshiftreturndata.append(errormsg);
+ redshiftreturndata.append(",");
+ redshiftreturndata.append(" \"results\": [] }");
+
+ return redshiftreturndata.toString();
+ }
+
+}
\ No newline at end of file
diff --git a/database/redshift/src/test/java/ThalesAWSRedshiftCRDPFPETester.java b/database/redshift/src/test/java/ThalesAWSRedshiftCRDPFPETester.java
new file mode 100644
index 00000000..a532ea88
--- /dev/null
+++ b/database/redshift/src/test/java/ThalesAWSRedshiftCRDPFPETester.java
@@ -0,0 +1,423 @@
+
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Logger;
+
+import okhttp3.MediaType;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+
+/*
+ * This test app to test the logic for a AWS Redshift Database User Defined
+ * Function(UDF). It is an example of how to use Thales Cipher REST Data Protection (CRDP)
+ * to protect sensitive data in a column. This example uses
+ * Format Preserve Encryption (FPE) to maintain the original format of the data
+ * so applications or business intelligence tools do not have to change in order
+ * to use these columns. There is no need to deploy a function to run it.
+ *
+ * Note: This source code is only to be used for testing and proof of concepts.
+ * Not production ready code. Was tested with CM 2.14 &
+ * CRDP 1.0 tech preview For more information on CRDP see link below.
+ * https://thalesdocs.com/ctp/con/crdp/latest/admin/index.html
+ *
+ * @author mwarner
+ *
+ */
+
+public class ThalesAWSRedshiftCRDPFPETester implements RequestStreamHandler {
+ private static final Logger logger = Logger.getLogger(ThalesAWSRedshiftCRDPFPETester.class.getName());
+
+ private static final String BADDATATAG = new String("9999999999999999");
+
+ private static final String REVEALRETURNTAG = new String("data");
+ private static final String PROTECTRETURNTAG = new String("protected_data");
+
+ private static final Gson gson = new Gson();
+
+ public static void main(String[] args) throws Exception {
+
+ ThalesAWSRedshiftCRDPFPETester nw2 = new ThalesAWSRedshiftCRDPFPETester();
+
+ String protect_internal_request = "{\r\n" + " \"request_id\" : \"23FF1F97-F28A-44AA-AB67-266ED976BF40\",\r\n"
+ + " \"cluster\" : \"arn:aws:redshift:xxxx\",\r\n" + " \"user\" : \"adminuser\",\r\n"
+ + " \"database\" : \"db1\",\r\n" + " \"external_function\": \"public.foo\",\r\n"
+ + " \"query_id\" : 5678234,\r\n" + " \"num_records\" : 5,\r\n" + " \"arguments\" : [\r\n"
+ + " [ \"5678234\"],\r\n" + " [ \"45345366345\"],\r\n" + " [ \"2342342342424\"],\r\n"
+ + " [ \"24\"],\r\n" + " [ \"5\"]\r\n" + " ]\r\n" + " }";
+
+ String revealrequest_internal = "{\r\n" + " \"request_id\" : \"23FF1F97-F28A-44AA-AB67-266ED976BF40\",\r\n"
+ + " \"cluster\" : \"arn:aws:redshift:xxxx\",\r\n" + " \"user\" : \"admin\",\r\n"
+ + " \"database\" : \"db1\",\r\n" + " \"external_function\": \"public.foo\",\r\n"
+ + " \"query_id\" : 5678234,\r\n" + " \"num_records\" : 5,\r\n" + " \"arguments\" : [\r\n"
+ + " [ \"10010012159021\"],\r\n" + " [ \"100100163267367633\"],\r\n"
+ + " [ \"10010014546420247261\"],\r\n" + " [ \"100100102\"],\r\n"
+ + " [ \"1004001ly4A5IY1j7QRDti4A2\"]\r\n" + " ]\r\n" + " }";
+
+ String protectrequest = "{\r\n" + " \"request_id\" : \"23FF1F97-F28A-44AA-AB67-266ED976BF40\",\r\n"
+ + " \"cluster\" : \"arn:aws:redshift:xxxx\",\r\n" + " \"user\" : \"adminuser\",\r\n"
+ + " \"database\" : \"db1\",\r\n" + " \"external_function\": \"public.foo\",\r\n"
+ + " \"query_id\" : 5678234,\r\n" + " \"num_records\" : 5,\r\n" + " \"arguments\" : [\r\n"
+ + " [ \"5678234\"],\r\n" + " [ \"45345366345\"],\r\n" + " [ \"2342342342424\"],\r\n"
+ + " [ \"2424\"],\r\n" + " [ \"234234234255667777\"]\r\n" + " ]\r\n" + " }";
+
+ String revealrequest = "{\r\n" + " \"request_id\" : \"23FF1F97-F28A-44AA-AB67-266ED976BF40\",\r\n"
+ + " \"cluster\" : \"arn:aws:redshift:xxxx\",\r\n" + " \"user\" : \"admin\",\r\n"
+ + " \"database\" : \"db1\",\r\n" + " \"external_function\": \"public.foo\",\r\n"
+ + " \"query_id\" : 5678234,\r\n" + " \"num_records\" : 5,\r\n" + " \"arguments\" : [\r\n"
+ + " [ \"1004001jKVC6JO\"],\r\n" + " [ \"1004001DW9FaQxLPj3\"],\r\n"
+ + " [ \"1004001rJivEodgILZg8\"],\r\n" + " [ \"1004001JvcC\"],\r\n"
+ + " [ \"1004001ly4A5IY1j7QRDti4A2\"]\r\n" + " ]\r\n" + " }";
+
+ String revealrequest_ext = "{\r\n" + " \"request_id\" : \"23FF1F97-F28A-44AA-AB67-266ED976BF40\",\r\n"
+ + " \"cluster\" : \"arn:aws:redshift:xxxx\",\r\n" + " \"user\" : \"admin\",\r\n"
+ + " \"database\" : \"db1\",\r\n" + " \"external_function\": \"public.foo\",\r\n"
+ + " \"query_id\" : 5678234,\r\n" + " \"num_records\" : 5,\r\n" + " \"arguments\" : [\r\n"
+ + " [ \"LmCJFjr\"],\r\n" + " [ \"SsoB9Lalcga\"],\r\n" + " [ \"p2TryIkXAjCLk\"],\r\n"
+ + " [ \"J5IX\"],\r\n" + " [ \"cWENACcmC12zWDxNpq\"]\r\n" + " ]\r\n" + " }";
+
+ String response = null;
+ String testJson = null;
+ nw2.handleRequest(protect_internal_request, null, null);
+
+ }
+
+ public void handleRequest(String inputStream, String outputStream, Context context)
+ throws IOException, CustomException {
+ Map reshift_ErrorMap = new HashMap();
+ // String input = IOUtils.toString(inputStream, "UTF-8");
+ JsonParser parser = new JsonParser();
+
+ int statusCode = 200;
+
+ String encdata = "";
+ String redshiftreturnstring = null;
+ String sensitive = null;
+
+ // StringBuffer redshiftreturndata = new StringBuffer();
+
+ boolean status = true;
+
+ JsonObject redshiftinput = null;
+ JsonElement rootNode = parser.parse(inputStream);
+ JsonArray redshiftdata = null;
+ String redshiftuserstr = null;
+
+ if (rootNode.isJsonObject()) {
+ redshiftinput = rootNode.getAsJsonObject();
+ if (redshiftinput != null) {
+ redshiftdata = redshiftinput.getAsJsonArray("arguments");
+ JsonPrimitive userjson = redshiftinput.getAsJsonPrimitive("user");
+ redshiftuserstr = userjson.getAsJsonPrimitive().toString();
+ redshiftuserstr = redshiftuserstr.replace("\"", "");
+ } else {
+ System.out.println("Root node not found.");
+ }
+ } else {
+ System.out.println("Bad data from snowflake.");
+ }
+
+ JsonPrimitive nbr_of_rows_json = redshiftinput.getAsJsonPrimitive("num_records");
+ String nbr_of_rows_json_str = nbr_of_rows_json.getAsJsonPrimitive().toString();
+ int nbr_of_rows_json_int = Integer.parseInt(nbr_of_rows_json_str);
+
+ String crdpip = System.getenv("CRDPIP");
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned
+ // when the user above does not have access to the key and if doing a
+ // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no
+ // then an error will be returned to the query, else the results set will provide ciphertext.
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ // yes,no
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB
+ // yes,no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be
+ // a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ ;
+ String keymetadatalocation = System.getenv("keymetadatalocation");
+ String external_version_from_ext_source = System.getenv("keymetadata");
+ String protection_profile = System.getenv("protection_profile");
+ String mode = System.getenv("mode");
+ String datatype = System.getenv("datatype");
+
+ String dataKey = null;
+ String protectedData = null;
+ String externalkeymetadata = null;
+ String crdpjsonBody = null;
+ boolean bad_data = false;
+
+ JsonObject result = new JsonObject();
+ JsonArray replies = new JsonArray();
+
+ // String external_version_from_ext_source = "1004001";
+ // String external_version_from_ext_source = "1001001";
+
+ JsonObject crdp_payload = new JsonObject();
+
+ crdp_payload.addProperty("protection_policy_name", protection_profile);
+ String jsonTagForProtectReveal = null;
+
+ String showrevealkey = "yes";
+
+ if (mode.equals("protect")) {
+ dataKey = "data";
+ jsonTagForProtectReveal = PROTECTRETURNTAG;
+ if (keymetadatalocation.equalsIgnoreCase("internal")) {
+ showrevealkey = System.getenv("showrevealinternalkey");
+ if (showrevealkey == null)
+ showrevealkey = "yes";
+ }
+ } else {
+ dataKey = "protected_data";
+ jsonTagForProtectReveal = REVEALRETURNTAG;
+ }
+ boolean showrevealkeybool = showrevealkey.equalsIgnoreCase("yes");
+
+ int nbrofrecords = redshiftdata.size();
+
+ try {
+
+ if (usersetlookupbool) {
+ // make sure cmuser is in Application Data Protection Clients Group
+
+ boolean founduserinuserset = findUserInUserSet(redshiftuserstr, userName, password, usersetID,
+ userSetLookupIP);
+ // System.out.println("Found User " + founduserinuserset);
+ if (!founduserinuserset)
+ throw new CustomException("1001, User Not in User Set", 1001);
+
+ } else {
+ usersetlookupbool = false;
+ }
+ OkHttpClient client = new OkHttpClient().newBuilder().build();
+ MediaType mediaType = MediaType.parse("application/json");
+ String urlStr = "http://" + crdpip + ":8090/v1/" + mode;
+
+ for (int i = 0; i < nbrofrecords; i++) {
+ JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
+
+ sensitive = checkValid(redshiftrow);
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+
+ }
+ encdata = sensitive;
+ } else {
+
+ crdp_payload.addProperty(dataKey, sensitive);
+ if (mode.equals("reveal")) {
+ crdp_payload.addProperty("username", redshiftuserstr);
+ if (keymetadatalocation.equalsIgnoreCase("external")) {
+ crdp_payload.addProperty("external_version", external_version_from_ext_source);
+ }
+ }
+ crdpjsonBody = crdp_payload.toString();
+
+ RequestBody body = RequestBody.create(mediaType, crdpjsonBody);
+
+ Request crdp_request = new Request.Builder()
+ // .url("http://192.168.159.143:8090/v1/protect").method("POST", body)
+ .url(urlStr).method("POST", body).addHeader("Content-Type", "application/json").build();
+ Response crdp_response = client.newCall(crdp_request).execute();
+
+ if (crdp_response.isSuccessful()) {
+
+ // Parse JSON response
+ String responseBody = crdp_response.body().string();
+ Gson gson = new Gson();
+ JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class);
+
+ if (jsonObject.has(jsonTagForProtectReveal)) {
+ protectedData = jsonObject.get(jsonTagForProtectReveal).getAsString();
+ if (keymetadatalocation.equalsIgnoreCase("external") && mode.equalsIgnoreCase("protect")) {
+ externalkeymetadata = jsonObject.get("external_version").getAsString();
+ System.out.println(
+ "Protected Data ext key metadata need to store this: " + externalkeymetadata);
+ }
+
+ if (keymetadatalocation.equalsIgnoreCase("internal") && mode.equalsIgnoreCase("protect")
+ && !showrevealkeybool) {
+ if (protectedData.length() > 7)
+ protectedData = protectedData.substring(7);
+ }
+
+ } else if (jsonObject.has("error_message")) {
+ String errorMessage = jsonObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ reshift_ErrorMap.put(i, errorMessage);
+ bad_data = true;
+ } else
+ System.out.println("unexpected json value from results: ");
+
+ } else {
+ System.err.println("Request failed with status code: " + crdp_response.code());
+ }
+
+ crdp_response.close();
+
+ encdata = protectedData;
+
+ }
+ replies.add(encdata);
+ } // end for loop
+
+ result.addProperty("success", true);
+ result.addProperty("num_records", nbrofrecords);
+ result.add("results", replies);
+
+ redshiftreturnstring = result.toString();
+
+ } catch (Exception e) {
+ System.out.println("in exception with " + e.getMessage());
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
+
+ for (int i = 0; i < redshiftdata.size(); i++) {
+ JsonArray redshiftrow = redshiftdata.get(i).getAsJsonArray();
+
+ JsonPrimitive redshiftcolumn = redshiftrow.get(0).getAsJsonPrimitive();
+
+ sensitive = redshiftcolumn.getAsJsonPrimitive().toString();
+ replies.add(sensitive);
+
+ }
+ result.addProperty("success", true);
+ result.addProperty("num_records", nbrofrecords);
+ result.add("results", replies);
+ redshiftreturnstring = result.toString();
+
+ } else {
+ statusCode = 400;
+ redshiftreturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+
+ } else {
+ statusCode = 400;
+ redshiftreturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ }
+
+ finally {
+
+ }
+
+ if (bad_data) {
+ System.out.println("errors: ");
+ for (Map.Entry entry : reshift_ErrorMap.entrySet()) {
+ Integer mkey = entry.getKey();
+ String mvalue = entry.getValue();
+ System.out.println("Error index: " + mkey);
+ System.out.println("Error Message: " + mvalue);
+ }
+
+ }
+
+ System.out.println(redshiftreturnstring);
+ // outputStream.write(new Gson().toJson(redshiftreturnstring).getBytes());
+ }
+
+ public String checkValid(JsonArray redshiftrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (redshiftrow != null && redshiftrow.size() > 0) {
+ JsonElement element = redshiftrow.get(0);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ public static String formatString(String inputString) {
+ // Split the input string to isolate the array content
+ String[] parts = inputString.split("\\[")[1].split("\\]")[0].split(",");
+
+ // Reformat the array elements to enclose them within double quotes
+ StringBuilder formattedArray = new StringBuilder();
+ for (String part : parts) {
+ formattedArray.append("\"").append(part.trim()).append("\",");
+ }
+
+ // Build the final formatted string
+ return inputString.replaceFirst("\\[.*?\\]",
+ "[" + formattedArray.deleteCharAt(formattedArray.length() - 1) + "]");
+ }
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer redshiftreturndata = new StringBuffer();
+
+ String errormsg = "\"Error in UDF \"";
+ redshiftreturndata.append("{ \"success\":");
+ redshiftreturndata.append(false);
+ redshiftreturndata.append(" \"num_records\":");
+ redshiftreturndata.append(0);
+ redshiftreturndata.append(",");
+ redshiftreturndata.append(" \"error_msg\":");
+ redshiftreturndata.append(errormsg);
+ redshiftreturndata.append(",");
+ redshiftreturndata.append(" \"results\": [] }");
+
+ return redshiftreturndata.toString();
+ }
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+ return cmuserset.findUserInUserSet(userName, newtoken);
+ }
+
+ @Override
+ public void handleRequest(InputStream input, OutputStream output, Context context) throws IOException {
+ // TODO Auto-generated method stub
+
+ }
+}
diff --git a/database/snowflake/CADP-SNOW-AWS-Functions/pom.xml b/database/snowflake/CADP-SNOW-AWS-Functions/pom.xml
index f3dcdeeb..d36083fd 100644
--- a/database/snowflake/CADP-SNOW-AWS-Functions/pom.xml
+++ b/database/snowflake/CADP-SNOW-AWS-Functions/pom.xml
@@ -103,36 +103,6 @@
-
- org.apache.maven.plugins
- maven-shade-plugin
- 3.2.2
-
- false
-
-
-
- package
-
- shade
-
-
-
-
-
- org.apache.maven.plugins
- maven-install-plugin
- 3.0.1
-
-
- Thales
- clean
-
- install-file
-
-
-
-
org.apache.maven.plugins
maven-shade-plugin
diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/python/ThalesGCPSnowCTVLTokenize.py b/database/snowflake/CADP-SNOW-GCP-Functions/python/ThalesGCPSnowCTVLTokenize.py
deleted file mode 100644
index d5d616e8..00000000
--- a/database/snowflake/CADP-SNOW-GCP-Functions/python/ThalesGCPSnowCTVLTokenize.py
+++ /dev/null
@@ -1,99 +0,0 @@
-# This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It uses CT-VL
-# to protect sensitive data in a column. This example will tokenize the data.
-#
-# Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
-# for all possible data sizes and combinations of encryption algorithms and IV, etc.
-# Was tested with CM 2.11 & CT-VL 2.6 or higher.
-# For more information on Snowflake External Functions see link below.
-# https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp
-#
-# @author mwarner
-#
-#
-
-
-import sys
-import requests
-import json
-import os
-#from requests.packages.urllib3.exceptions import InsecureRequestWarning
-from urllib3 import disable_warnings
-from urllib3.exceptions import InsecureRequestWarning
-disable_warnings(InsecureRequestWarning)
-
-
-HTTP_SUCCESS = 200
-HTTP_FAILURE = 400
-# Declare the variable
-
-p11ptext = None
-p11auth = None
-
-p11debug = True
-p11user = os.environ.get('P11USER')
-p11url = os.environ.get('P11URL')
-p11tokgroup = "tg1"
-p11toktemplate = "tt1"
-p11ptext = "thisistestdata"
-# Loop through the option, set value to variable
-
-p11url = "https://" + p11url + "/vts"
-# Check username and password if exist in system environment or not
-
-# split the p11user variable to check username/password missing
-p11auth = tuple(p11user.split(":", 1))
-
-errs = 0
-
-# Set the header application
-hdrs = {'Content-Type': 'application/json'}
-requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
-
-############ Check connect to VTS before continue #######################
-try:
- r = requests.get(p11url, verify=False)
- r.raise_for_status()
-except requests.exceptions.RequestException as err:
- print("\nEror, something wrong: ")
- print(err)
- sys.exit(200)
-except requests.exceptions.HTTPError as errh:
- print("\nHttp Error:", errh)
- sys.exit(200)
-except requests.exceptions.ConnectionError as errc:
- print("\nError Connecting:", errc)
- sys.exit(200)
-except requests.exceptions.Timeout as errt:
- print("\nTimeout Error:", errt)
- sys.exit(200)
-
-################################################################################
-
-def thales_cts_tokenize(request):
-
- try:
- # The list of rows to return.
- return_value = []
- result = None
- payload = request.get_json()
-
- rows = payload["data"]
- u = p11url + "/rest/v2.0/tokenize"
- # For each input row
- for row in rows:
- # Include the row number.
- row_number = row[0]
- row_value = row[1]
- data = {"tokengroup" : p11tokgroup, "tokentemplate" : p11toktemplate}
- data["data"] = row_value
- # Post the request
- r = requests.post(u, headers=hdrs, auth=p11auth, verify=False, json=data)
- result = json.loads(r.text)
- row_to_return = [row_number, result['token']]
- return_value.append(row_to_return)
- json_compatible_string_to_return = json.dumps( { "data" : return_value } )
- return (json_compatible_string_to_return, HTTP_SUCCESS)
-
- except:
-
- return(request.data, HTTP_FAILURE)
\ No newline at end of file
diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/python/requirements.txt b/database/snowflake/CADP-SNOW-GCP-Functions/python/requirements.txt
deleted file mode 100644
index 8d0f3557..00000000
--- a/database/snowflake/CADP-SNOW-GCP-Functions/python/requirements.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-# Function dependencies, for example:
-# package>=version
-functions-framework
-requests==2.31.0
-urllib3==1.26.8
\ No newline at end of file
diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharDecryptBulkFPE.java b/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharDecryptBulkFPE.java
deleted file mode 100644
index 35e24eb6..00000000
--- a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharDecryptBulkFPE.java
+++ /dev/null
@@ -1,222 +0,0 @@
-package com.example;
-
-import javax.crypto.Cipher;
-import com.google.cloud.functions.HttpFunction;
-import com.google.cloud.functions.HttpRequest;
-import com.google.cloud.functions.HttpResponse;
-import com.ingrian.security.nae.IngrianProvider;
-import com.ingrian.security.nae.AbstractNAECipher;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec;
-import com.ingrian.security.nae.NAEKey;
-import com.ingrian.security.nae.NAESession;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
-import com.ingrian.security.nae.IngrianProvider.Builder;
-import com.ingrian.security.nae.NAECipher;
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonPrimitive;
-
-import java.security.spec.AlgorithmParameterSpec;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.logging.Logger;
-
-/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application
- * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
- * data so applications or business intelligence tools do not have to change in order to use these columns.
-*
-* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
-* for all possible data sizes and combinations of encryption algorithms and IV, etc.
-* Was tested with CM 2.11 & CADP 8.13
-* For more information on CADP see link below.
-https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html
-* For more information on Snowflake External Functions see link below.
-https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp
-
-Notes: This example uses the CADP bulk API.
-Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data
-element has Unicode characters then the supported size limit would be 1750 characters
-
-Size of spec array should be same as the number of elements if user wants to use separate spec values
-for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data
-index.
- *
- *@author mwarner
- *
- */
-
-public class ThalesGCPSnowCADPCharDecryptBulkFPE implements HttpFunction {
-// @Override
- private static byte[][] data;
- private static AlgorithmParameterSpec[] spec;
- private static Integer[] snowrownbrs;
-
- private static int BATCHLIMIT = 10000;
- private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPCharDecryptBulkFPE.class.getName());
- private static final Gson gson = new Gson();
-
- public void service(HttpRequest request, HttpResponse response) throws Exception {
-
- String tweakAlgo = null;
- String tweakData = null;
- String snowflakereturnstring = null;
- JsonArray snowflakedata = null;
- NAESession session = null;
-
- Map encryptedErrorMapTotal = new HashMap();
- try {
-
- String keyName = "testfaas";
- String userName = System.getenv("CMUSER");
- String password = System.getenv("CMPWD");
- int batchsize = Integer.parseInt(System.getenv("BATCHSIZE"));
- // JsonArray snowflakedata = null;
-
- if (batchsize >= BATCHLIMIT)
- batchsize = BATCHLIMIT;
- spec = new FPEParameterAndFormatSpec[batchsize];
- data = new byte[batchsize][];
- snowrownbrs = new Integer[batchsize];
-
- try {
- JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class);
- JsonObject requestJson = null;
-
- if (requestParsed != null && requestParsed.isJsonObject()) {
- requestJson = requestParsed.getAsJsonObject();
- }
-
- if (requestJson != null && requestJson.has("data")) {
- snowflakedata = requestJson.getAsJsonArray("data");
-
- }
-
- } catch (JsonParseException e) {
- logger.severe("Error parsing JSON: " + e.getMessage());
- }
-
- // System.setProperty("com.ingrian.security.nae.IngrianNAE_Properties_Conf_Filename",
- // "IngrianNAE.properties");
- System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
- "CADP_for_JAVA.properties");
- IngrianProvider builder = new Builder().addConfigFileInputStream(
- getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
- // IngrianProvider builder = new
- // Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build();
- session = NAESession.getSession(userName, password.toCharArray());
- NAEKey key = NAEKey.getSecretKey(keyName, session);
- int row_number = 0;
-
- StringBuffer snowflakereturndata = new StringBuffer();
- // Serialization
- snowflakereturndata.append("{ \"data\": [");
- String algorithm = "FPE/FF1/CARD62";
-
- AbstractNAECipher decryptCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider");
- int i = 0;
-
- int totalRowsLeft = snowflakedata.size();
- while (i < snowflakedata.size()) {
- int index = 0;
- int dataIndex = 0;
- int specIndex = 0;
- int snowRowIndex = 0;
- // System.out.println("totalRowsLeft begining =" + totalRowsLeft);
- if (totalRowsLeft < batchsize) {
- spec = new FPEParameterAndFormatSpec[totalRowsLeft];
- data = new byte[totalRowsLeft][];
- snowrownbrs = new Integer[totalRowsLeft];
- } else {
- spec = new FPEParameterAndFormatSpec[batchsize];
- data = new byte[batchsize][];
- snowrownbrs = new Integer[batchsize];
- }
-
- for (int b = 0; b < batchsize && b < totalRowsLeft; b++) {
-
- JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
-
- for (int j = 0; j < snowflakerow.size(); j++) {
-
- JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
- // System.out.print(snowflakecolumn + " ");
- String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
- // get a cipher
- if (j == 1) {
-
- // FPE example
- data[dataIndex++] = sensitive.getBytes();
- spec[specIndex++] = new FPEParameterAndFormatBuilder(tweakData)
- .set_tweakAlgorithm(tweakAlgo).build();
- } else {
-
- row_number = snowflakecolumn.getAsInt();
- snowrownbrs[snowRowIndex++] = row_number;
-
- }
-
- }
-
- i++;
- }
- // make bulk call....
- // initializing the cipher for encrypt operation
- decryptCipher.init(Cipher.DECRYPT_MODE, key, spec[0]);
-
- // Map to store exceptions while encryption
- Map decryptErrorMap = new HashMap();
-
- // performing bulk operation
- byte[][] decryptedData = decryptCipher.doFinalBulk(data, spec, decryptErrorMap);
-
- for (Map.Entry entry : decryptErrorMap.entrySet()) {
- Integer mkey = entry.getKey();
- String mvalue = entry.getValue();
- encryptedErrorMapTotal.put(mkey, mvalue);
- }
-
- for (int enc = 0; enc < decryptedData.length; enc++) {
-
- snowflakereturndata.append("[");
- snowflakereturndata.append(snowrownbrs[enc]);
- snowflakereturndata.append(",");
- snowflakereturndata.append(new String(decryptedData[enc]));
-
- if (index <= batchsize - 1 || index < totalRowsLeft - 1)
- // if (index < batchsize -1 && index < totalRowsLeft -1 )
- {
- if (totalRowsLeft - 1 > 0)
- snowflakereturndata.append("],");
- else
- snowflakereturndata.append("]");
- } else {
-
- snowflakereturndata.append("]");
- }
-
- index++;
- totalRowsLeft--;
-
- }
- }
-
- snowflakereturndata.append("]}");
-
- snowflakereturnstring = new String(snowflakereturndata);
-
- } catch (Exception e) {
- // return "check exception";
- }
- finally{
- if(session!=null) {
- session.closeSession();
- }
- }
-
- response.getWriter().write(snowflakereturnstring);
-
- }
-}
\ No newline at end of file
diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharDecryptFPE.java b/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharDecryptFPE.java
deleted file mode 100644
index 4692b1e4..00000000
--- a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharDecryptFPE.java
+++ /dev/null
@@ -1,147 +0,0 @@
-package com.example;
-
-import javax.crypto.Cipher;
-import com.google.cloud.functions.HttpFunction;
-import com.google.cloud.functions.HttpRequest;
-import com.google.cloud.functions.HttpResponse;
-import com.ingrian.security.nae.IngrianProvider;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec;
-import com.ingrian.security.nae.NAEKey;
-import com.ingrian.security.nae.NAESession;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
-import com.ingrian.security.nae.IngrianProvider.Builder;
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonPrimitive;
-import java.util.logging.Logger;
-
-/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application
- * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
- * data so applications or business intelligence tools do not have to change in order to use these columns.
-*
-* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
-* for all possible data sizes and combinations of encryption algorithms and IV, etc.
-* Was tested with CM 2.11 & CADP 8.13
-* For more information on CADP see link below.
-https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html
-* For more information on Snowflake External Functions see link below.
-https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp
- *
- *@author mwarner
- *
- */
-
-public class ThalesGCPSnowCADPCharDecryptFPE implements HttpFunction {
-// @Override
-
- private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPCharDecryptFPE.class.getName());
- private static final Gson gson = new Gson();
-
- public void service(HttpRequest request, HttpResponse response) throws Exception {
-
- String decData = "";
- String tweakAlgo = null;
- String tweakData = null;
- String snowflakereturnstring = null;
- NAESession session = null;
-
- try {
-
- String keyName = "testfaas";
- String userName = System.getenv("CMUSER");
- String password = System.getenv("CMPWD");
- JsonArray snowflakedata = null;
-
- try {
- JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class);
- JsonObject requestJson = null;
-
- if (requestParsed != null && requestParsed.isJsonObject()) {
- requestJson = requestParsed.getAsJsonObject();
- }
-
- if (requestJson != null && requestJson.has("data")) {
- snowflakedata = requestJson.getAsJsonArray("data");
-
- }
-
- } catch (JsonParseException e) {
- logger.severe("Error parsing JSON: " + e.getMessage());
- }
-
- // System.setProperty("com.ingrian.security.nae.IngrianNAE_Properties_Conf_Filename",
- // "IngrianNAE.properties");
- System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
- "CADP_for_JAVA.properties");
- IngrianProvider builder = new Builder().addConfigFileInputStream(
- getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
- // IngrianProvider builder = new
- // Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build();
- session = NAESession.getSession(userName, password.toCharArray());
- NAEKey key = NAEKey.getSecretKey(keyName, session);
- int row_number = 0;
-
- StringBuffer snowflakereturndata = new StringBuffer();
- // Serialization
- snowflakereturndata.append("{ \"data\": [");
- String algorithm = "FPE/FF1/CARD62";
- // String algorithm = "AES/CBC/PKCS5Padding";
- FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
- .build();
-
- Cipher decryptCipher = Cipher.getInstance(algorithm, "IngrianProvider");
-
- for (int i = 0; i < snowflakedata.size(); i++) {
- JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
- snowflakereturndata.append("[");
- for (int j = 0; j < snowflakerow.size(); j++) {
-
- JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
- String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
- // get a cipher
- if (j == 1) {
-
- // FPE example
-
- // initialize cipher to encrypt.
- decryptCipher.init(Cipher.DECRYPT_MODE, key, param);
- // encrypt data
- byte[] outbuf = decryptCipher.doFinal(sensitive.getBytes());
- decData = new String(outbuf);
-
- snowflakereturndata.append(decData);
-
- } else {
- row_number = snowflakecolumn.getAsInt();
- snowflakereturndata.append(row_number);
- snowflakereturndata.append(",");
- }
-
- }
- if (snowflakedata.size() == 1 || i == snowflakedata.size() - 1)
- snowflakereturndata.append("]");
- else
- snowflakereturndata.append("],");
-
- }
-
- snowflakereturndata.append("]}");
-
- snowflakereturnstring = new String(snowflakereturndata);
-
- } catch (Exception e) {
- // return "check exception";
- }
- finally{
- if(session!=null) {
- session.closeSession();
- }
- }
-
- response.getWriter().write(snowflakereturnstring);
-
- }
-}
\ No newline at end of file
diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharEncryptBulkFPE.java b/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharEncryptBulkFPE.java
deleted file mode 100644
index f8398982..00000000
--- a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharEncryptBulkFPE.java
+++ /dev/null
@@ -1,219 +0,0 @@
-package com.example;
-
-import javax.crypto.Cipher;
-import com.google.cloud.functions.HttpFunction;
-import com.google.cloud.functions.HttpRequest;
-import com.google.cloud.functions.HttpResponse;
-import com.ingrian.security.nae.IngrianProvider;
-import com.ingrian.security.nae.AbstractNAECipher;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec;
-import com.ingrian.security.nae.NAEKey;
-import com.ingrian.security.nae.NAESession;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
-import com.ingrian.security.nae.IngrianProvider.Builder;
-import com.ingrian.security.nae.NAECipher;
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonPrimitive;
-
-import java.security.spec.AlgorithmParameterSpec;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.logging.Logger;
-/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application
- * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
- * data so applications or business intelligence tools do not have to change in order to use these columns.
-*
-* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
-* for all possible data sizes and combinations of encryption algorithms and IV, etc.
-* Was tested with CM 2.11 & CADP 8.13
-* For more information on CADP see link below.
-https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html
-* For more information on Snowflake External Functions see link below.
-https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp
-
-Notes: This example uses the CADP bulk API.
-Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data
-element has Unicode characters then the supported size limit would be 1750 characters
-
-Size of spec array should be same as the number of elements if user wants to use separate spec values
-for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data
-index.
- *
- *@author mwarner
- *
- */
-public class ThalesGCPSnowCADPCharEncryptBulkFPE implements HttpFunction {
-// @Override
- private static byte[][] data;
- private static AlgorithmParameterSpec[] spec;
- private static Integer[] snowrownbrs;
-
- private static int BATCHLIMIT = 10000;
- private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPCharEncryptBulkFPE.class.getName());
- private static final Gson gson = new Gson();
-
- public void service(HttpRequest request, HttpResponse response) throws Exception {
-
- String tweakAlgo = null;
- String tweakData = null;
- String snowflakereturnstring = null;
- JsonArray snowflakedata = null;
- NAESession session = null;
-
-
- Map encryptedErrorMapTotal = new HashMap();
- try {
-
- String keyName = "testfaas";
- String userName = System.getenv("CMUSER");
- String password = System.getenv("CMPWD");
- int batchsize = Integer.parseInt(System.getenv("BATCHSIZE"));
- if (batchsize >= BATCHLIMIT)
- batchsize = BATCHLIMIT;
- spec = new FPEParameterAndFormatSpec[batchsize];
- data = new byte[batchsize][];
- snowrownbrs = new Integer[batchsize];
-
- try {
- JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class);
- JsonObject requestJson = null;
-
- if (requestParsed != null && requestParsed.isJsonObject()) {
- requestJson = requestParsed.getAsJsonObject();
- }
-
- if (requestJson != null && requestJson.has("data")) {
- snowflakedata = requestJson.getAsJsonArray("data");
-
- }
-
- } catch (JsonParseException e) {
- logger.severe("Error parsing JSON: " + e.getMessage());
- }
-
- // System.setProperty("com.ingrian.security.nae.IngrianNAE_Properties_Conf_Filename",
- // "IngrianNAE.properties");
- System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
- "CADP_for_JAVA.properties");
- IngrianProvider builder = new Builder().addConfigFileInputStream(
- getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
- // IngrianProvider builder = new
- // Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build();
- session = NAESession.getSession(userName, password.toCharArray());
- NAEKey key = NAEKey.getSecretKey(keyName, session);
- int row_number = 0;
-
- StringBuffer snowflakereturndata = new StringBuffer();
- // Serialization
- snowflakereturndata.append("{ \"data\": [");
- String algorithm = "FPE/FF1/CARD62";
-
- AbstractNAECipher encryptCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider");
- int i = 0;
-
- int totalRowsLeft = snowflakedata.size();
- while (i < snowflakedata.size()) {
- int index = 0;
- int dataIndex = 0;
- int specIndex = 0;
- int snowRowIndex = 0;
- // System.out.println("totalRowsLeft begining =" + totalRowsLeft);
- if (totalRowsLeft < batchsize) {
- spec = new FPEParameterAndFormatSpec[totalRowsLeft];
- data = new byte[totalRowsLeft][];
- snowrownbrs = new Integer[totalRowsLeft];
- } else {
- spec = new FPEParameterAndFormatSpec[batchsize];
- data = new byte[batchsize][];
- snowrownbrs = new Integer[batchsize];
- }
-
- for (int b = 0; b < batchsize && b < totalRowsLeft; b++) {
-
- JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
-
- for (int j = 0; j < snowflakerow.size(); j++) {
-
- JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
- // System.out.print(snowflakecolumn + " ");
- String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
- // get a cipher
- if (j == 1) {
-
- // FPE example
- data[dataIndex++] = sensitive.getBytes();
- spec[specIndex++] = new FPEParameterAndFormatBuilder(tweakData)
- .set_tweakAlgorithm(tweakAlgo).build();
- } else {
-
- row_number = snowflakecolumn.getAsInt();
- snowrownbrs[snowRowIndex++] = row_number;
-
- }
-
- }
-
- i++;
- }
- // make bulk call....
- // initializing the cipher for encrypt operation
- encryptCipher.init(Cipher.ENCRYPT_MODE, key, spec[0]);
-
- // Map to store exceptions while encryption
- Map encryptedErrorMap = new HashMap();
-
- // performing bulk operation
- byte[][] encryptedData = encryptCipher.doFinalBulk(data, spec, encryptedErrorMap);
-
- for (Map.Entry entry : encryptedErrorMap.entrySet()) {
- Integer mkey = entry.getKey();
- String mvalue = entry.getValue();
- encryptedErrorMapTotal.put(mkey, mvalue);
- }
-
- for (int enc = 0; enc < encryptedData.length; enc++) {
-
- snowflakereturndata.append("[");
- snowflakereturndata.append(snowrownbrs[enc]);
- snowflakereturndata.append(",");
- snowflakereturndata.append(new String(encryptedData[enc]));
-
- if (index <= batchsize - 1 || index < totalRowsLeft - 1)
- // if (index < batchsize -1 && index < totalRowsLeft -1 )
- {
- if (totalRowsLeft - 1 > 0)
- snowflakereturndata.append("],");
- else
- snowflakereturndata.append("]");
- } else {
-
- snowflakereturndata.append("]");
- }
-
- index++;
- totalRowsLeft--;
-
- }
- }
-
- snowflakereturndata.append("]}");
-
- snowflakereturnstring = new String(snowflakereturndata);
-
- } catch (Exception e) {
- // return "check exception";
- }
- finally{
- if(session!=null) {
- session.closeSession();
- }
- }
-
- response.getWriter().write(snowflakereturnstring);
-
- }
-}
\ No newline at end of file
diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharEncryptFPE.java b/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharEncryptFPE.java
deleted file mode 100644
index 9740f080..00000000
--- a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPCharEncryptFPE.java
+++ /dev/null
@@ -1,144 +0,0 @@
-package com.example;
-
-import javax.crypto.Cipher;
-import com.google.cloud.functions.HttpFunction;
-import com.google.cloud.functions.HttpRequest;
-import com.google.cloud.functions.HttpResponse;
-import com.ingrian.security.nae.IngrianProvider;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec;
-import com.ingrian.security.nae.NAEKey;
-import com.ingrian.security.nae.NAESession;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
-import com.ingrian.security.nae.IngrianProvider.Builder;
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonPrimitive;
-import java.util.logging.Logger;
-/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application
- * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
- * data so applications or business intelligence tools do not have to change in order to use these columns.
-*
-* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
-* for all possible data sizes and combinations of encryption algorithms and IV, etc.
-* Was tested with CM 2.11 & CADP 8.13
-* For more information on CADP see link below.
-https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html
-* For more information on Snowflake External Functions see link below.
-https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp
- *
- *@author mwarner
- *
- */
-public class ThalesGCPSnowCADPCharEncryptFPE implements HttpFunction {
-// @Override
-
- private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPCharEncryptFPE.class.getName());
- private static final Gson gson = new Gson();
-
- public void service(HttpRequest request, HttpResponse response) throws Exception {
-
- String encdata = "";
- String tweakAlgo = null;
- String tweakData = null;
- String snowflakereturnstring = null;
- NAESession session = null;
- try {
-
- String keyName = "testfaas";
- String userName = System.getenv("CMUSER");
- String password = System.getenv("CMPWD");
- JsonArray snowflakedata = null;
-
- try {
- JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class);
- JsonObject requestJson = null;
-
- if (requestParsed != null && requestParsed.isJsonObject()) {
- requestJson = requestParsed.getAsJsonObject();
- }
-
- if (requestJson != null && requestJson.has("data")) {
- snowflakedata = requestJson.getAsJsonArray("data");
-
- }
-
- } catch (JsonParseException e) {
- logger.severe("Error parsing JSON: " + e.getMessage());
- }
-
- // System.setProperty("com.ingrian.security.nae.IngrianNAE_Properties_Conf_Filename",
- // "IngrianNAE.properties");
- System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
- "CADP_for_JAVA.properties");
- IngrianProvider builder = new Builder().addConfigFileInputStream(
- getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
- // IngrianProvider builder = new
- // Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build();
- session = NAESession.getSession(userName, password.toCharArray());
- NAEKey key = NAEKey.getSecretKey(keyName, session);
- int row_number = 0;
-
- StringBuffer snowflakereturndata = new StringBuffer();
- // Serialization
- snowflakereturndata.append("{ \"data\": [");
- String algorithm = "FPE/FF1/CARD62";
- // String algorithm = "AES/CBC/PKCS5Padding";
- FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
- .build();
-
- Cipher encryptCipher = Cipher.getInstance(algorithm, "IngrianProvider");
-
- for (int i = 0; i < snowflakedata.size(); i++) {
- JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
- snowflakereturndata.append("[");
- for (int j = 0; j < snowflakerow.size(); j++) {
-
- JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
- String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
- // get a cipher
- if (j == 1) {
-
- // FPE example
-
- // initialize cipher to encrypt.
- encryptCipher.init(Cipher.ENCRYPT_MODE, key, param);
- // encrypt data
- byte[] outbuf = encryptCipher.doFinal(sensitive.getBytes());
- encdata = new String(outbuf);
-
- snowflakereturndata.append(encdata);
-
- } else {
- row_number = snowflakecolumn.getAsInt();
- snowflakereturndata.append(row_number);
- snowflakereturndata.append(",");
- }
-
- }
- if (snowflakedata.size() == 1 || i == snowflakedata.size() - 1)
- snowflakereturndata.append("]");
- else
- snowflakereturndata.append("],");
-
- }
-
- snowflakereturndata.append("]}");
-
- snowflakereturnstring = new String(snowflakereturndata);
-
- } catch (Exception e) {
- // return "check exception";
- }
- finally{
- if(session!=null) {
- session.closeSession();
- }
- }
-
- response.getWriter().write(snowflakereturnstring);
-
- }
-}
\ No newline at end of file
diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrDecryptBulkFPE.java b/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrDecryptBulkFPE.java
deleted file mode 100644
index 55f24572..00000000
--- a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrDecryptBulkFPE.java
+++ /dev/null
@@ -1,218 +0,0 @@
-package com.example;
-
-import javax.crypto.Cipher;
-import com.google.cloud.functions.HttpFunction;
-import com.google.cloud.functions.HttpRequest;
-import com.google.cloud.functions.HttpResponse;
-import com.ingrian.security.nae.IngrianProvider;
-import com.ingrian.security.nae.AbstractNAECipher;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec;
-import com.ingrian.security.nae.NAEKey;
-import com.ingrian.security.nae.NAESession;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
-import com.ingrian.security.nae.IngrianProvider.Builder;
-import com.ingrian.security.nae.NAECipher;
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonPrimitive;
-
-import java.security.spec.AlgorithmParameterSpec;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.logging.Logger;
-
-/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application
- * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
- * data so applications or business intelligence tools do not have to change in order to use these columns.
-*
-* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
-* for all possible data sizes and combinations of encryption algorithms and IV, etc.
-* Was tested with CM 2.11 & CADP 8.13
-* For more information on CADP see link below.
-https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html
-* For more information on Snowflake External Functions see link below.
-https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp
-
-Notes: This example uses the CADP bulk API.
-Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data
-element has Unicode characters then the supported size limit would be 1750 characters
-
-Size of spec array should be same as the number of elements if user wants to use separate spec values
-for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data
-index.
- *
- *@author mwarner
- *
- */
-public class ThalesGCPSnowCADPNbrDecryptBulkFPE implements HttpFunction {
-// @Override
- private static byte[][] data;
- private static AlgorithmParameterSpec[] spec;
- private static Integer[] snowrownbrs;
-
- private static int BATCHLIMIT = 10000;
- private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPNbrDecryptBulkFPE.class.getName());
- private static final Gson gson = new Gson();
-
- public void service(HttpRequest request, HttpResponse response) throws Exception {
-
- String tweakAlgo = null;
- String tweakData = null;
- String snowflakereturnstring = null;
- JsonArray snowflakedata = null;
- NAESession session = null;
- Map encryptedErrorMapTotal = new HashMap();
- try {
-
- String keyName = "testfaas";
- String userName = System.getenv("CMUSER");
- String password = System.getenv("CMPWD");
- int batchsize = Integer.parseInt(System.getenv("BATCHSIZE")); ;
- if (batchsize >= BATCHLIMIT)
- batchsize = BATCHLIMIT;
- spec = new FPEParameterAndFormatSpec[batchsize];
- data = new byte[batchsize][];
- snowrownbrs = new Integer[batchsize];
-
- try {
- JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class);
- JsonObject requestJson = null;
-
- if (requestParsed != null && requestParsed.isJsonObject()) {
- requestJson = requestParsed.getAsJsonObject();
- }
-
- if (requestJson != null && requestJson.has("data")) {
- snowflakedata = requestJson.getAsJsonArray("data");
-
- }
-
- } catch (JsonParseException e) {
- logger.severe("Error parsing JSON: " + e.getMessage());
- }
-
- // System.setProperty("com.ingrian.security.nae.IngrianNAE_Properties_Conf_Filename",
- // "IngrianNAE.properties");
- System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
- "CADP_for_JAVA.properties");
- IngrianProvider builder = new Builder().addConfigFileInputStream(
- getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
- // IngrianProvider builder = new
- // Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build();
- session = NAESession.getSession(userName, password.toCharArray());
- NAEKey key = NAEKey.getSecretKey(keyName, session);
- int row_number = 0;
-
- StringBuffer snowflakereturndata = new StringBuffer();
- // Serialization
- snowflakereturndata.append("{ \"data\": [");
- String algorithm = "FPE/FF1/CARD10";
-
- AbstractNAECipher decryptCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider");
- int i = 0;
-
- int totalRowsLeft = snowflakedata.size();
- while (i < snowflakedata.size()) {
- int index = 0;
- int dataIndex = 0;
- int specIndex = 0;
- int snowRowIndex = 0;
- // System.out.println("totalRowsLeft begining =" + totalRowsLeft);
- if (totalRowsLeft < batchsize) {
- spec = new FPEParameterAndFormatSpec[totalRowsLeft];
- data = new byte[totalRowsLeft][];
- snowrownbrs = new Integer[totalRowsLeft];
- } else {
- spec = new FPEParameterAndFormatSpec[batchsize];
- data = new byte[batchsize][];
- snowrownbrs = new Integer[batchsize];
- }
-
- for (int b = 0; b < batchsize && b < totalRowsLeft; b++) {
-
- JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
-
- for (int j = 0; j < snowflakerow.size(); j++) {
-
- JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
- // System.out.print(snowflakecolumn + " ");
- String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
- // get a cipher
- if (j == 1) {
-
- // FPE example
- data[dataIndex++] = sensitive.getBytes();
- spec[specIndex++] = new FPEParameterAndFormatBuilder(tweakData)
- .set_tweakAlgorithm(tweakAlgo).build();
- } else {
-
- row_number = snowflakecolumn.getAsInt();
- snowrownbrs[snowRowIndex++] = row_number;
-
- }
-
- }
-
- i++;
- }
- // make bulk call....
- // initializing the cipher for encrypt operation
- decryptCipher.init(Cipher.DECRYPT_MODE, key, spec[0]);
-
- // Map to store exceptions while encryption
- Map decryptErrorMap = new HashMap();
-
- // performing bulk operation
- byte[][] decryptedData = decryptCipher.doFinalBulk(data, spec, decryptErrorMap);
-
- for (Map.Entry entry : decryptErrorMap.entrySet()) {
- Integer mkey = entry.getKey();
- String mvalue = entry.getValue();
- encryptedErrorMapTotal.put(mkey, mvalue);
- }
-
- for (int enc = 0; enc < decryptedData.length; enc++) {
-
- snowflakereturndata.append("[");
- snowflakereturndata.append(snowrownbrs[enc]);
- snowflakereturndata.append(",");
- snowflakereturndata.append(new String(decryptedData[enc]));
-
- if (index <= batchsize - 1 || index < totalRowsLeft - 1)
- // if (index < batchsize -1 && index < totalRowsLeft -1 )
- {
- if (totalRowsLeft - 1 > 0)
- snowflakereturndata.append("],");
- else
- snowflakereturndata.append("]");
- } else {
-
- snowflakereturndata.append("]");
- }
-
- index++;
- totalRowsLeft--;
-
- }
- }
-
- snowflakereturndata.append("]}");
-
- snowflakereturnstring = new String(snowflakereturndata);
-
- } catch (Exception e) {
- // return "check exception";
- }
- finally{
- if(session!=null) {
- session.closeSession();
- }
- }
-
- response.getWriter().write(snowflakereturnstring);
-
- }
-}
\ No newline at end of file
diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrDecryptFPE.java b/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrDecryptFPE.java
deleted file mode 100644
index c67ef8c6..00000000
--- a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrDecryptFPE.java
+++ /dev/null
@@ -1,146 +0,0 @@
-package com.example;
-
-
-import javax.crypto.Cipher;
-import com.google.cloud.functions.HttpFunction;
-import com.google.cloud.functions.HttpRequest;
-import com.google.cloud.functions.HttpResponse;
-import com.ingrian.security.nae.IngrianProvider;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec;
-import com.ingrian.security.nae.NAEKey;
-import com.ingrian.security.nae.NAESession;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
-import com.ingrian.security.nae.IngrianProvider.Builder;
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonPrimitive;
-import java.util.logging.Logger;
-
-/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application
- * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
- * data so applications or business intelligence tools do not have to change in order to use these columns.
-*
-* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
-* for all possible data sizes and combinations of encryption algorithms and IV, etc.
-* Was tested with CM 2.11 & CADP 8.13
-* For more information on CADP see link below.
-https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html
-* For more information on Snowflake External Functions see link below.
-https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp
- *
- *@author mwarner
- *
- */
-public class ThalesGCPSnowCADPNbrDecryptFPE implements HttpFunction {
-// @Override
-
- private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPNbrDecryptFPE.class.getName());
- private static final Gson gson = new Gson();
- public void service(HttpRequest request, HttpResponse response)
- throws Exception {
-
- String decData = "";
- String tweakAlgo = null;
- String tweakData = null;
- String snowflakereturnstring = null;
- NAESession session = null;
-
- try {
-
- String keyName = "testfaas";
- String userName = System.getenv("CMUSER");
- String password = System.getenv("CMPWD");
- JsonArray snowflakedata = null;
-
- try {
- JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class);
- JsonObject requestJson = null;
-
- if (requestParsed != null && requestParsed.isJsonObject()) {
- requestJson = requestParsed.getAsJsonObject();
- }
-
- if (requestJson != null && requestJson.has("data")) {
- snowflakedata = requestJson.getAsJsonArray("data");
-
- }
-
- } catch (JsonParseException e) {
- logger.severe("Error parsing JSON: " + e.getMessage());
- }
-
- // System.setProperty("com.ingrian.security.nae.IngrianNAE_Properties_Conf_Filename", "IngrianNAE.properties");
- System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", "CADP_for_JAVA.properties");
- IngrianProvider builder = new Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
- //IngrianProvider builder = new Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build();
- session = NAESession.getSession(userName, password.toCharArray());
- NAEKey key = NAEKey.getSecretKey(keyName, session);
-
- int row_number = 0;
-
- StringBuffer snowflakereturndata = new StringBuffer();
- // Serialization
- snowflakereturndata.append("{ \"data\": [");
- String algorithm = "FPE/FF1/CARD10";
- // String algorithm = "AES/CBC/PKCS5Padding";
- FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
- .build();
-
- Cipher decryptCipher = Cipher.getInstance(algorithm, "IngrianProvider");
-
- for (int i = 0; i < snowflakedata.size(); i++) {
- JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
- snowflakereturndata.append("[");
- for (int j = 0; j < snowflakerow.size(); j++) {
-
- JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
-
- String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
- // get a cipher
- if (j == 1) {
-
- // FPE example
-
- // initialize cipher to encrypt.
- decryptCipher.init(Cipher.DECRYPT_MODE, key, param);
- // encrypt data
- byte[] outbuf = decryptCipher.doFinal(sensitive.getBytes());
- decData = new String(outbuf);
- //System.out.println("Enc data : " + encdata);
-
- snowflakereturndata.append(decData);
-
- } else {
- row_number = snowflakecolumn.getAsInt();
- snowflakereturndata.append(row_number);
- snowflakereturndata.append(",");
- }
-
- }
- if ( snowflakedata.size() == 1 || i == snowflakedata.size() -1)
- snowflakereturndata.append("]");
- else
- snowflakereturndata.append("],");
-
- }
-
- snowflakereturndata.append("]}");
-
- snowflakereturnstring = new String(snowflakereturndata);
-
- } catch (Exception e) {
- // return "check exception";
- }
- finally{
- if(session!=null) {
- session.closeSession();
- }
- }
-
- response.getWriter().write(snowflakereturnstring);
-
- }
-}
\ No newline at end of file
diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrEncryptBulkFPE.java b/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrEncryptBulkFPE.java
deleted file mode 100644
index 2d68d011..00000000
--- a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrEncryptBulkFPE.java
+++ /dev/null
@@ -1,220 +0,0 @@
-package com.example;
-
-import javax.crypto.Cipher;
-import com.google.cloud.functions.HttpFunction;
-import com.google.cloud.functions.HttpRequest;
-import com.google.cloud.functions.HttpResponse;
-import com.ingrian.security.nae.IngrianProvider;
-import com.ingrian.security.nae.AbstractNAECipher;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec;
-import com.ingrian.security.nae.NAEKey;
-import com.ingrian.security.nae.NAESession;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
-import com.ingrian.security.nae.IngrianProvider.Builder;
-import com.ingrian.security.nae.NAECipher;
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonPrimitive;
-
-import java.security.spec.AlgorithmParameterSpec;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.logging.Logger;
-
-/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application
- * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
- * data so applications or business intelligence tools do not have to change in order to use these columns.
-*
-* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
-* for all possible data sizes and combinations of encryption algorithms and IV, etc.
-* Was tested with CM 2.11 & CADP 8.13
-* For more information on CADP see link below.
-https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html
-* For more information on Snowflake External Functions see link below.
-https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp
-
-Notes: This example uses the CADP bulk API.
-Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data
-element has Unicode characters then the supported size limit would be 1750 characters
-
-Size of spec array should be same as the number of elements if user wants to use separate spec values
-for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data
-index.
- *
- *@author mwarner
- *
- */
-public class ThalesGCPSnowCADPNbrEncryptBulkFPE implements HttpFunction {
-// @Override
- private static byte[][] data;
- private static AlgorithmParameterSpec[] spec;
- private static Integer[] snowrownbrs;
-
- private static int BATCHLIMIT = 10000;
- private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPNbrEncryptBulkFPE.class.getName());
- private static final Gson gson = new Gson();
-
- public void service(HttpRequest request, HttpResponse response) throws Exception {
-
- String tweakAlgo = null;
- String tweakData = null;
- String snowflakereturnstring = null;
- JsonArray snowflakedata = null;
- Map encryptedErrorMapTotal = new HashMap();
-
- NAESession session = null;
- try {
-
- String keyName = "testfaas";
- String userName = System.getenv("CMUSER");
- String password = System.getenv("CMPWD");
- int batchsize = Integer.parseInt(System.getenv("BATCHSIZE"));
- if (batchsize >= BATCHLIMIT)
- batchsize = BATCHLIMIT;
- spec = new FPEParameterAndFormatSpec[batchsize];
- data = new byte[batchsize][];
- snowrownbrs = new Integer[batchsize];
-
- try {
- JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class);
- JsonObject requestJson = null;
-
- if (requestParsed != null && requestParsed.isJsonObject()) {
- requestJson = requestParsed.getAsJsonObject();
- }
-
- if (requestJson != null && requestJson.has("data")) {
- snowflakedata = requestJson.getAsJsonArray("data");
-
- }
-
- } catch (JsonParseException e) {
- logger.severe("Error parsing JSON: " + e.getMessage());
- }
-
- // System.setProperty("com.ingrian.security.nae.IngrianNAE_Properties_Conf_Filename",
- // "IngrianNAE.properties");
- System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
- "CADP_for_JAVA.properties");
- IngrianProvider builder = new Builder().addConfigFileInputStream(
- getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
- // IngrianProvider builder = new
- // Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build();
- session = NAESession.getSession(userName, password.toCharArray());
- NAEKey key = NAEKey.getSecretKey(keyName, session);
- int row_number = 0;
-
- StringBuffer snowflakereturndata = new StringBuffer();
- // Serialization
- snowflakereturndata.append("{ \"data\": [");
- String algorithm = "FPE/FF1/CARD10";
-
- AbstractNAECipher encryptCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider");
- int i = 0;
-
- int totalRowsLeft = snowflakedata.size();
- while (i < snowflakedata.size()) {
- int index = 0;
- int dataIndex = 0;
- int specIndex = 0;
- int snowRowIndex = 0;
- // System.out.println("totalRowsLeft begining =" + totalRowsLeft);
- if (totalRowsLeft < batchsize) {
- spec = new FPEParameterAndFormatSpec[totalRowsLeft];
- data = new byte[totalRowsLeft][];
- snowrownbrs = new Integer[totalRowsLeft];
- } else {
- spec = new FPEParameterAndFormatSpec[batchsize];
- data = new byte[batchsize][];
- snowrownbrs = new Integer[batchsize];
- }
-
- for (int b = 0; b < batchsize && b < totalRowsLeft; b++) {
-
- JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
-
- for (int j = 0; j < snowflakerow.size(); j++) {
-
- JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
- // System.out.print(snowflakecolumn + " ");
- String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
- // get a cipher
- if (j == 1) {
-
- // FPE example
- data[dataIndex++] = sensitive.getBytes();
- spec[specIndex++] = new FPEParameterAndFormatBuilder(tweakData)
- .set_tweakAlgorithm(tweakAlgo).build();
- } else {
-
- row_number = snowflakecolumn.getAsInt();
- snowrownbrs[snowRowIndex++] = row_number;
-
- }
-
- }
-
- i++;
- }
- // make bulk call....
- // initializing the cipher for encrypt operation
- encryptCipher.init(Cipher.ENCRYPT_MODE, key, spec[0]);
-
- // Map to store exceptions while encryption
- Map encryptedErrorMap = new HashMap();
-
- // performing bulk operation
- byte[][] encryptedData = encryptCipher.doFinalBulk(data, spec, encryptedErrorMap);
-
- for (Map.Entry entry : encryptedErrorMap.entrySet()) {
- Integer mkey = entry.getKey();
- String mvalue = entry.getValue();
- encryptedErrorMapTotal.put(mkey, mvalue);
- }
-
- for (int enc = 0; enc < encryptedData.length; enc++) {
-
- snowflakereturndata.append("[");
- snowflakereturndata.append(snowrownbrs[enc]);
- snowflakereturndata.append(",");
- snowflakereturndata.append(new String(encryptedData[enc]));
-
- if (index <= batchsize - 1 || index < totalRowsLeft - 1)
- // if (index < batchsize -1 && index < totalRowsLeft -1 )
- {
- if (totalRowsLeft - 1 > 0)
- snowflakereturndata.append("],");
- else
- snowflakereturndata.append("]");
- } else {
-
- snowflakereturndata.append("]");
- }
-
- index++;
- totalRowsLeft--;
-
- }
- }
-
- snowflakereturndata.append("]}");
-
- snowflakereturnstring = new String(snowflakereturndata);
-
-
- } catch (Exception e) {
- // return "check exception";
- }
- finally{
- if(session!=null) {
- session.closeSession();
- }
- }
-
- response.getWriter().write(snowflakereturnstring);
-
- }
-}
\ No newline at end of file
diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrEncryptFPE.java b/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrEncryptFPE.java
deleted file mode 100644
index 03357e0d..00000000
--- a/database/snowflake/CADP-SNOW-GCP-Functions/src/main/java/com/example/ThalesGCPSnowCADPNbrEncryptFPE.java
+++ /dev/null
@@ -1,144 +0,0 @@
-package com.example;
-
-
-import javax.crypto.Cipher;
-import com.google.cloud.functions.HttpFunction;
-import com.google.cloud.functions.HttpRequest;
-import com.google.cloud.functions.HttpResponse;
-import com.ingrian.security.nae.IngrianProvider;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec;
-import com.ingrian.security.nae.NAEKey;
-import com.ingrian.security.nae.NAESession;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
-import com.ingrian.security.nae.IngrianProvider.Builder;
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonPrimitive;
-import java.util.logging.Logger;
-
-/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application
- * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
- * data so applications or business intelligence tools do not have to change in order to use these columns.
-*
-* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
-* for all possible data sizes and combinations of encryption algorithms and IV, etc.
-* Was tested with CM 2.11 & CADP 8.13
-* For more information on CADP see link below.
-https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html
-* For more information on Snowflake External Functions see link below.
-https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp
- *
- *@author mwarner
- *
- */
-public class ThalesGCPSnowCADPNbrEncryptFPE implements HttpFunction {
-// @Override
-
- private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPNbrEncryptFPE.class.getName());
- private static final Gson gson = new Gson();
- public void service(HttpRequest request, HttpResponse response)
- throws Exception {
-
- String encdata = "";
- String tweakAlgo = null;
- String tweakData = null;
- String snowflakereturnstring = null;
- NAESession session = null;
- try {
-
- String keyName = "testfaas";
- String userName = System.getenv("CMUSER");
- String password = System.getenv("CMPWD");
- JsonArray snowflakedata = null;
-
- try {
- JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class);
- JsonObject requestJson = null;
-
- if (requestParsed != null && requestParsed.isJsonObject()) {
- requestJson = requestParsed.getAsJsonObject();
- }
-
- if (requestJson != null && requestJson.has("data")) {
- snowflakedata = requestJson.getAsJsonArray("data");
-
- }
-
- } catch (JsonParseException e) {
- logger.severe("Error parsing JSON: " + e.getMessage());
- }
-
-
- System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", "CADP_for_JAVA.properties");
- IngrianProvider builder = new Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
- //IngrianProvider builder = new Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build();
- session = NAESession.getSession(userName, password.toCharArray());
- NAEKey key = NAEKey.getSecretKey(keyName, session);
- int row_number = 0;
-
- StringBuffer snowflakereturndata = new StringBuffer();
- // Serialization
- snowflakereturndata.append("{ \"data\": [");
- String algorithm = "FPE/FF1/CARD10";
- // String algorithm = "AES/CBC/PKCS5Padding";
- FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
- .build();
-
- Cipher encryptCipher = Cipher.getInstance(algorithm, "IngrianProvider");
-
- for (int i = 0; i < snowflakedata.size(); i++) {
- JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
- snowflakereturndata.append("[");
- for (int j = 0; j < snowflakerow.size(); j++) {
-
- JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
-
- String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
- // get a cipher
- if (j == 1) {
-
- // FPE example
-
- // initialize cipher to encrypt.
- encryptCipher.init(Cipher.ENCRYPT_MODE, key, param);
- // encrypt data
- byte[] outbuf = encryptCipher.doFinal(sensitive.getBytes());
- encdata = new String(outbuf);
-
- snowflakereturndata.append(encdata);
-
- } else {
- row_number = snowflakecolumn.getAsInt();
- snowflakereturndata.append(row_number);
- snowflakereturndata.append(",");
- }
-
- }
- if ( snowflakedata.size() == 1 || i == snowflakedata.size() -1)
- snowflakereturndata.append("]");
- else
- snowflakereturndata.append("],");
-
- }
-
- snowflakereturndata.append("]}");
-
- snowflakereturnstring = new String(snowflakereturndata);
-
-
- } catch (Exception e) {
- // return "check exception";
- }
- finally{
- if(session!=null) {
- session.closeSession();
- }
- }
-
- response.getWriter().write(snowflakereturnstring);
-
- }
-}
\ No newline at end of file
diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/src/test/java/GCPCloudFunctionTester.java b/database/snowflake/CADP-SNOW-GCP-Functions/src/test/java/GCPCloudFunctionTester.java
deleted file mode 100644
index dff482a3..00000000
--- a/database/snowflake/CADP-SNOW-GCP-Functions/src/test/java/GCPCloudFunctionTester.java
+++ /dev/null
@@ -1,173 +0,0 @@
-
-
-
-import javax.crypto.Cipher;
-
-import com.google.cloud.functions.HttpFunction;
-import com.google.cloud.functions.HttpRequest;
-import com.google.cloud.functions.HttpResponse;
-import com.ingrian.security.nae.IngrianProvider;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec;
-import com.ingrian.security.nae.NAEKey;
-import com.ingrian.security.nae.NAESession;
-import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
-import com.ingrian.security.nae.IngrianProvider.Builder;
-import com.google.gson.Gson;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonPrimitive;
-import java.util.logging.Logger;
-
-/* This test app to test the logic for a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application
- * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
- * data so applications or business intelligence tools do not have to change in order to use these columns. There is no need to deploy a function to run it.
-*
-* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
-* for all possible data sizes and combinations of encryption algorithms and IV, etc.
-* Was tested with CM 2.11 & CADP 8.13
-* For more information on CADP see link below.
-https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html
-* For more information on Snowflake External Functions see link below.
-https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp
- *
- *@author mwarner
- *
- */
-public class GCPCloudFunctionTester implements HttpFunction {
-// @Override
-
- private static final Logger logger = Logger.getLogger(GCPCloudFunctionTester.class.getName());
- private static final Gson gson = new Gson();
-
- public static void main(String[] args) throws Exception
- {
-
- GCPCloudFunctionTester nw2 = new GCPCloudFunctionTester();
-
- String request = "{ \"data\":\r\n" +
- " [\r\n" +
- " [ 0, \"page\" ],\r\n" +
- " [ 1, \"life, the universe, and everything\" ]\r\n" +
- " ]\r\n" +
- "}";
-
- String response = null;
- nw2.service(request, response);
-
-
-
- }
-
-
- public void service(String request, String response)
- throws Exception {
-
- String encdata = "";
- String tweakAlgo = null;
- String tweakData = null;
- String snowflakereturnstring = null;
- NAESession session = null;
- try {
-
- String keyName = "testfaas";
- String userName = System.getenv("CMUSER");
- String password = System.getenv("CMPWD");
- JsonArray snowflakedata = null;
-
- try {
- JsonElement requestParsed = gson.fromJson(request, JsonElement.class);
- JsonObject requestJson = null;
-
- if (requestParsed != null && requestParsed.isJsonObject()) {
- requestJson = requestParsed.getAsJsonObject();
- }
-
- if (requestJson != null && requestJson.has("data")) {
- snowflakedata = requestJson.getAsJsonArray("data");
-
- }
-
- } catch (JsonParseException e) {
- logger.severe("Error parsing JSON: " + e.getMessage());
- }
-
-
- System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename", "CADP_for_JAVA.properties");
- IngrianProvider builder = new Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
- //IngrianProvider builder = new Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build();
- session = NAESession.getSession(userName, password.toCharArray());
- NAEKey key = NAEKey.getSecretKey(keyName, session);
- int row_number = 0;
-
- StringBuffer snowflakereturndata = new StringBuffer();
- // Serialization
- snowflakereturndata.append("{ \"data\": [");
- String algorithm = "FPE/FF1/CARD62";
- // String algorithm = "AES/CBC/PKCS5Padding";
- FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
- .build();
-
- Cipher encryptCipher = Cipher.getInstance(algorithm, "IngrianProvider");
-
- for (int i = 0; i < snowflakedata.size(); i++) {
- JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
- snowflakereturndata.append("[");
- for (int j = 0; j < snowflakerow.size(); j++) {
-
- JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
-
- String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
- // get a cipher
- if (j == 1) {
-
- // FPE example
-
- // initialize cipher to encrypt.
- encryptCipher.init(Cipher.ENCRYPT_MODE, key, param);
- // encrypt data
- byte[] outbuf = encryptCipher.doFinal(sensitive.getBytes());
- encdata = new String(outbuf);
-
- snowflakereturndata.append(encdata);
-
- } else {
- row_number = snowflakecolumn.getAsInt();
- snowflakereturndata.append(row_number);
- snowflakereturndata.append(",");
- }
-
- }
- if ( snowflakedata.size() == 1 || i == snowflakedata.size() -1)
- snowflakereturndata.append("]");
- else
- snowflakereturndata.append("],");
-
- }
-
- snowflakereturndata.append("]}");
-
- snowflakereturnstring = new String(snowflakereturndata);
-
-
- } catch (Exception e) {
- // return "check exception";
- }
- finally{
- if(session!=null) {
- session.closeSession();
- }
- }
- System.out.println(snowflakereturnstring);
- //response.getWriter().write(snowflakereturnstring);
-
- }
-
-
-@Override
-public void service(HttpRequest request, HttpResponse response) throws Exception {
- // TODO Auto-generated method stub
-
-}
-}
\ No newline at end of file
diff --git a/database/snowflake/README.md b/database/snowflake/README.md
index 51135de1..9a49983c 100644
--- a/database/snowflake/README.md
+++ b/database/snowflake/README.md
@@ -1,4 +1,4 @@
# Integration with Snowflake using User-Defined Functions
-CADP-SNOW-AWS-Functions is for AWS Lambda
-CADP-SNOW-GCP-Functions is for GCP Functions
-
+CADP-SNOW-AWS-Functions is for AWS Lambda (initial versions)
+Thales-Snow-AWS-UDF - is the newer AWS versions that support better error handling and fewer classes with more env variables.
+Thales-Snow-GCP-UDF - GCP UDFs that support better error handling and fewer classes with more env variables.
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-AWS-UDF/pom.xml b/database/snowflake/Thales-Snow-AWS-UDF/pom.xml
new file mode 100644
index 00000000..01693b5c
--- /dev/null
+++ b/database/snowflake/Thales-Snow-AWS-UDF/pom.xml
@@ -0,0 +1,147 @@
+
+ 4.0.0
+ Thales
+ Thales-Snow-AWS-UDF
+ 0.0.3-SNAPSHOT
+
+
+ 11
+ 11
+ 3.12.0
+ 4.4
+ 1.70
+ 1.10
+
+ 31.1-jre
+ 2.9.0
+ 2.17.2
+ 2.17.2
+ .000
+
+
+
+
+
+ io.github.thalescpl-io.cadp
+ CADP_for_JAVA
+ 8.15.0.001
+
+
+
+ org.apache.commons
+ commons-lang3
+ 3.12.0
+
+
+ com.squareup.okhttp3
+ okhttp
+ 4.10.0
+
+
+ javax.xml.bind
+ jaxb-api
+ 2.3.1
+
+
+
+ commons-codec
+ commons-codec
+ ${org.apache.commons.codec.version}
+
+
+ com.google.guava
+ guava
+ ${guava.version}
+
+
+ com.google.code.findbugs
+ jsr305
+
+
+ com.google.errorprone
+ error_prone_annotations
+
+
+ com.google.guava
+ listenablefuture
+
+
+ com.google.j2objc
+ j2objc-annotations
+
+
+ org.checkerframework
+ checker-qual
+
+
+
+
+ com.google.code.gson
+ gson
+ ${gson.version}
+
+
+ org.apache.commons
+ commons-collections4
+ 4.4
+
+
+ commons-io
+ commons-io
+ 2.14.0
+
+
+ com.amazonaws
+ aws-lambda-java-core
+ 1.2.1
+
+
+ com.amazonaws
+ aws-lambda-java-events
+ 3.11.0
+
+
+ com.amazonaws
+ aws-lambda-java-log4j2
+ 1.5.1
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+ 3.2.2
+
+ false
+
+
+
+ package
+
+ shade
+
+
+
+
+
+
+
+
+
+
+
+ com.github.edwgiz
+ maven-shade-plugin.log4j2-cachefile-transformer
+
+ 2.13.0
+
+
+
+
+
+ Thales-Snow-AWS-UDF
+
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/CMUserSetHelper.java b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/CMUserSetHelper.java
new file mode 100644
index 00000000..06f17015
--- /dev/null
+++ b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/CMUserSetHelper.java
@@ -0,0 +1,503 @@
+package com.example;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.security.KeyStore;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509TrustManager;
+/*
+ * This app provides a number of different helper methods dealing with CM Application Data Protection UserSets. There is a method to find
+ * a user in a userset and another method to populate the userset from a flat file. Usersets are typically used within
+ * an access policy but they are not restricted to that usage.
+ * Was tested with CM 2.14
+ * Note: This source code is only to be used for testing and proof of concepts.
+ * @author mwarner
+ *
+ */
+
+public class CMUserSetHelper {
+
+ static String hostnamevalidate = "yourhostname";
+
+ String usersetid = "716f01a6-5cab-4799-925a-6dc2d8712fc1";
+ String cmIP = "yourip";
+ String apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?name=";
+ // + username_in_userset;
+ String addusertouserset = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users";
+
+ int totalrecords = 0;
+ String authUrl = "https://" + cmIP + "/api/v1/auth/tokens";
+
+ static boolean debug = true;
+ int chunksize = 5;
+ static int CHUNKSIZEMAX = 100;
+
+ public CMUserSetHelper(String usersetid, String cmIP) {
+
+ this.usersetid = usersetid;
+ this.cmIP = cmIP;
+ this.apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?name=";
+ this.addusertouserset = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users";
+ this.authUrl = "https://" + cmIP + "/api/v1/auth/tokens";
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ String username = args[0];
+ String password = args[1];
+ String cmip = args[2];
+ String usersetid = args[3];
+ String filePath = args[4];
+ CMUserSetHelper cmusersetHelper = new CMUserSetHelper(usersetid, cmip);
+ int totalrecords = 0;
+
+ String jwthtoken = geAuthToken(cmusersetHelper.authUrl, username, password);
+
+ String newtoken = "Bearer " + removeQuotes(jwthtoken);
+
+ // String filePath =
+ // "C:\\Users\\t0185905\\workspace\\CT-VL-GCP\\src\\main\\java\\com\\example\\emailAddresses.txt";
+ RandomAccessFile file = new RandomAccessFile(filePath, "r");
+ if (cmusersetHelper.chunksize > CHUNKSIZEMAX)
+ cmusersetHelper.chunksize = CHUNKSIZEMAX;
+ int totoalnbrofrecords = numberOfLines(file);
+ if (cmusersetHelper.chunksize > totoalnbrofrecords) {
+ cmusersetHelper.chunksize = totoalnbrofrecords / 2;
+ }
+ //totalrecords = cmusersetHelper.addAUserToUserSet(cmusersetHelper.addusertouserset, newtoken);
+ totalrecords = cmusersetHelper.addAUserToUserSetFromFile(cmusersetHelper.addusertouserset, newtoken, filePath);
+ System.out.println("Totalrecords inserted into Userset " + cmusersetHelper.usersetid + " = " + totalrecords);
+
+ }
+
+ /**
+ * Returns an boolean if user found
+ *
+ *
+ * @param user user to find
+ * @param newtoken jwt token to use
+ * @return boolean true if found in userset
+ * @throws CustomException
+ */
+ public boolean findUserInUserSet(String user, String newtoken) throws CustomException {
+ boolean found = false;
+
+ String apiUrl = this.apiUrlGetUsers + user;
+
+ apiUrl = removeQuotes(apiUrl);
+
+ try {
+ URL url = new URL(apiUrl);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestMethod("GET");
+ connection.setRequestProperty("Authorization", newtoken);
+ connection.setRequestProperty("accept", "application/json");
+
+ connection.setDoInput(true);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+
+ StringBuilder response = new StringBuilder();
+ String line;
+ while ((line = reader.readLine()) != null) {
+ response.append(line);
+ }
+ reader.close();
+
+ Gson gson = new Gson();
+ if (debug)
+ System.out.println("response " + response);
+ JsonObject input = null;
+ JsonElement total = null;
+ JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ input = rootNode.getAsJsonObject();
+ if (input.isJsonObject()) {
+ total = input.get("total");
+ }
+ }
+ JsonPrimitive column = total.getAsJsonPrimitive();
+ String totalstr = column.getAsJsonPrimitive().toString();
+
+ Integer i = Integer.valueOf(totalstr);
+
+ if (i > 0) {
+ found = true;
+
+ }
+ connection.disconnect();
+ } catch (Exception e) {
+ if (e.getMessage().contains("403")) {
+ throw new CustomException("1002, User Not in Application Data Protection Clients ", 1002);
+ } else
+ e.printStackTrace();
+ }
+
+ return found;
+
+ }
+
+ /**
+ * Loads users from a file to the userset. Returns an int of number of users
+ * added.
+ *
+ *
+ * @param url url to userset api
+ * @param newtoken jwt token to use
+ * @param filePath file to load
+ * @return int totalnumberofrecords added to userset
+ */
+
+ public int addAUserToUserSetFromFile(String url, String newtoken, String filePath) throws IOException {
+
+ int totalnbrofrecords = 0;
+ try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
+ String line;
+ int count = 0;
+ StringBuilder payloadBuilder = new StringBuilder();
+
+ payloadBuilder.append("{\"users\": [");
+
+ while ((line = br.readLine()) != null) {
+ totalnbrofrecords++;
+ payloadBuilder.append("\"").append(line).append("\",");
+ count++;
+
+ // If 'n' records have been read, print the payload
+ if (count == chunksize) {
+ makeCMCall(payloadBuilder, newtoken, url);
+ // Reset payload builder and count for the next batch of records
+ payloadBuilder = new StringBuilder("{\"users\": [");
+ count = 0;
+ }
+ }
+
+ // If there are remaining records, print the payload
+ if (count > 0) {
+ payloadBuilder.deleteCharAt(payloadBuilder.length() - 1); // Remove the trailing comma
+ payloadBuilder.append("}");
+ makeCMCall(payloadBuilder, newtoken, url);
+ if (debug)
+ System.out.println(payloadBuilder.toString());
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ return totalnbrofrecords;
+
+ }
+
+ /**
+ * Loads users from a file to the userset. Returns an int of number of users
+ * added.
+ *
+ *
+ * @param payloadBuilder payload for CM call that contains users to add
+ * @param newtoken jwt token to use
+ * @param url url to peform inserts
+ * @return int response code
+ */
+
+ public static int makeCMCall(StringBuilder payloadBuilder, String newtoken, String url) throws IOException {
+
+ payloadBuilder.deleteCharAt(payloadBuilder.length() - 1); // Remove the trailing comma
+ payloadBuilder.append("]}");
+ if (debug)
+ System.out.println(payloadBuilder.toString());
+ String payload = payloadBuilder.toString();
+
+ // Create URL object
+ URL apiUrl = new URL(url);
+ // Create connection object
+ HttpURLConnection connection = (HttpURLConnection) apiUrl.openConnection();
+ connection.setRequestMethod("POST");
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setDoOutput(true);
+ connection.setRequestProperty("Authorization", newtoken);
+ // Send request
+ OutputStream outputStream = connection.getOutputStream();
+ outputStream.write(payload.getBytes());
+ outputStream.flush();
+ outputStream.close();
+
+ // Get response
+ int responseCode = connection.getResponseCode();
+ BufferedReader reader;
+ if (responseCode >= 200 && responseCode < 300) {
+ reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+ } else {
+ reader = new BufferedReader(new InputStreamReader(connection.getErrorStream()));
+ }
+
+ // Read response
+ String lineresponse;
+ StringBuilder response = new StringBuilder();
+ while ((lineresponse = reader.readLine()) != null) {
+ response.append(lineresponse);
+ }
+ reader.close();
+
+ // Print response
+ if (debug) {
+ System.out.println("Response Code: " + responseCode);
+ System.out.println("Response Body: " + response.toString());
+ }
+ // Disconnect connection
+ connection.disconnect();
+ // Reset payload builder and count for the next batch of records
+ payloadBuilder = new StringBuilder("{\"users\": [");
+
+ return responseCode;
+
+ }
+
+ /**
+ * Simple sample of showing how to load a couple of users to the userset.
+ * Returns an int of number of users added.
+ *
+ *
+ * @param url url to userset api
+ * @param newtoken jwt token to use
+ *
+ * @return int response code
+ */
+ public static int addAUserToUserSet(String url, String newtoken) throws IOException {
+ boolean found = false;
+
+ String payload = "{\"users\": [\"akhip@company.com\",\"user2@company.com\"]}";
+
+ // Create URL object
+ URL apiUrl = new URL(url);
+ // Create connection object
+ HttpURLConnection connection = (HttpURLConnection) apiUrl.openConnection();
+ connection.setRequestMethod("POST");
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setDoOutput(true);
+ connection.setRequestProperty("Authorization", newtoken);
+ // Send request
+ OutputStream outputStream = connection.getOutputStream();
+ outputStream.write(payload.getBytes());
+ outputStream.flush();
+ outputStream.close();
+
+ // Get response
+ int responseCode = connection.getResponseCode();
+ BufferedReader reader;
+ if (responseCode >= 200 && responseCode < 300) {
+ reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+ } else {
+ reader = new BufferedReader(new InputStreamReader(connection.getErrorStream()));
+ }
+
+ // Read response
+ String line;
+ StringBuilder response = new StringBuilder();
+ while ((line = reader.readLine()) != null) {
+ response.append(line);
+ }
+ reader.close();
+
+ // Print response
+ if (debug) {
+ System.out.println("Response Code: " + responseCode);
+ System.out.println("Response Body: " + response.toString());
+ }
+ // Disconnect connection
+ connection.disconnect();
+
+ return responseCode;
+
+ }
+
+ public static String removeQuotes(String input) {
+
+ // Remove double quotes from the input string
+ input = input.replace("\"", "");
+
+ return input;
+ }
+
+ private static int numberOfLines(RandomAccessFile file) throws IOException {
+ int numberOfLines = 0;
+ while (file.readLine() != null) {
+ numberOfLines++;
+ }
+ file.seek(0);
+ return numberOfLines;
+ }
+
+ /**
+ * Get JWT from CM
+ *
+ *
+ * @param apiUrl url to CM auth
+ * @param username username on CM
+ * @param pwd password on CM
+ * @return String jwt
+ */
+
+ public static String geAuthToken(String apiUrl, String usernb, String pwd) throws Exception
+
+ {
+
+ String jStr = "{\"username\":\"" + usernb + "\",\"password\":\"" + pwd + "\"}";
+ disableCertValidation();
+
+ String jwtstr = null;
+ try {
+ URL url = new URL(apiUrl);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestProperty("Content-length", String.valueOf(jStr.length()));
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setRequestMethod("GET");
+ connection.setDoOutput(true);
+ connection.setDoInput(true);
+ DataOutputStream output = new DataOutputStream(connection.getOutputStream());
+ output.writeBytes(jStr);
+ output.close();
+ BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+ StringBuilder response = new StringBuilder();
+ String line;
+ while ((line = reader.readLine()) != null) {
+ response.append(line);
+ }
+ reader.close();
+
+ JsonObject input = null;
+ JsonElement jwt = null;
+ JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ input = rootNode.getAsJsonObject();
+ if (input.isJsonObject()) {
+ jwt = input.get("jwt");
+ }
+ }
+ JsonPrimitive column = jwt.getAsJsonPrimitive();
+ jwtstr = column.getAsJsonPrimitive().toString();
+ connection.disconnect();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ return jwtstr;
+
+ }
+
+ /**
+ * Get JWT from CM
+ *
+ *
+ * @param apiUrl url to CM auth
+ * @param username username on CM
+ * @param pwd password on CM
+ * @return String jwt
+ */
+
+ public static String geAuthToken(String apiUrl) throws Exception
+
+ {
+
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+ String jStr = "{\"username\":\"" + userName + "\",\"password\":\"" + password + "\"}";
+
+ disableCertValidation();
+
+ String jwtstr = null;
+ try {
+ URL url = new URL(apiUrl);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestProperty("Content-length", String.valueOf(jStr.length()));
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setRequestMethod("GET");
+ connection.setDoOutput(true);
+ connection.setDoInput(true);
+ DataOutputStream output = new DataOutputStream(connection.getOutputStream());
+ output.writeBytes(jStr);
+ output.close();
+ BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+ StringBuilder response = new StringBuilder();
+ String line;
+ while ((line = reader.readLine()) != null) {
+ response.append(line);
+ }
+ reader.close();
+ if (debug)
+ System.out.println("response " + response);
+ JsonObject input = null;
+ JsonElement jwt = null;
+ JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ input = rootNode.getAsJsonObject();
+ if (input.isJsonObject()) {
+ jwt = input.get("jwt");
+ }
+ }
+ JsonPrimitive column = jwt.getAsJsonPrimitive();
+ jwtstr = column.getAsJsonPrimitive().toString();
+ connection.disconnect();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ return jwtstr;
+
+ }
+
+ public static void disableCertValidation() throws Exception {
+ TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() {
+ public java.security.cert.X509Certificate[] getAcceptedIssuers() {
+ return null;
+ }
+
+ public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) {
+ }
+
+ public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) {
+ }
+ } };
+
+ // Install the all-trusting trust manager
+ try {
+ SSLContext sc = SSLContext.getInstance("SSL");
+ sc.init(null, trustAllCerts, new java.security.SecureRandom());
+ HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ // Create all-trusting host name verifier
+ HostnameVerifier allHostsValid = new HostnameVerifier() {
+ public boolean verify(String hostname, SSLSession session) {
+ return true;
+ }
+ };
+
+ // Install the all-trusting host verifier
+ HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid);
+ }
+
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/CustomException.java b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/CustomException.java
new file mode 100644
index 00000000..9bf5066c
--- /dev/null
+++ b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/CustomException.java
@@ -0,0 +1,15 @@
+package example;
+
+
+class CustomException extends Exception {
+ private int errorCode;
+
+ public CustomException(String message, int errorCode) {
+ super(message);
+ this.errorCode = errorCode;
+ }
+
+ public int getErrorCode() {
+ return errorCode;
+ }
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCADPFPEBulkUDF.java b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCADPFPEBulkUDF.java
new file mode 100644
index 00000000..7449643c
--- /dev/null
+++ b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCADPFPEBulkUDF.java
@@ -0,0 +1,511 @@
+package example;
+
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.math.BigInteger;
+import java.security.spec.AlgorithmParameterSpec;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+
+import org.apache.commons.io.IOUtils;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+import com.ingrian.security.nae.AbstractNAECipher;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec;
+import com.ingrian.security.nae.IngrianProvider;
+import com.ingrian.security.nae.NAECipher;
+import com.ingrian.security.nae.NAEKey;
+import com.ingrian.security.nae.NAESession;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
+import com.ingrian.security.nae.IngrianProvider.Builder;
+
+/* This sample AWS Lambda Function is used to implement a Snowflake Database User Defined Function(UDF). /*
+ * It is an example of how to use Thales Cipher Trust Application Data Protection (FCADP)
+ * to protect sensitive data in a column. This example uses
+ * Format Preserve Encryption (FPE) to maintain the original format of the data
+ * so applications or business intelligence tools do not have to change in order
+ * to use these columns.
+*
+* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
+* for all possible data sizes and combinations of encryption algorithms and IV, etc.
+* Was tested with CM 2.14 & CADP CADP 8.15.0.001
+* For more information on CADP see link below.
+https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html
+* For more information on Snowflake External Functions see link below.
+https://docs.snowflake.com/en/sql-reference/external-functions-creating-aws
+
+Notes: This example uses the CADP bulk API.
+Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data
+element has Unicode characters then the supported size limit would be 1750 characters
+
+Size of spec array should be same as the number of elements if user wants to use separate spec values
+for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data
+index.
+ *
+ * @author mwarner
+ */
+
+public class ThalesAWSSnowCADPFPEBulkUDF implements RequestStreamHandler {
+
+ private static byte[][] data;
+ private static AlgorithmParameterSpec[] spec;
+ private static Integer[] snowrownbrs;
+ private static final String BADDATATAG = new String("9999999999999999");
+ private static int BATCHLIMIT = 10000;
+
+ // private static final Logger logger =
+ // Logger.getLogger(LambdaRequestStreamHandlerNbrEncyrptBulkFPE.class.getName());
+ private static final Gson gson = new Gson();
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+
+ boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
+
+ return founduserinuserset;
+
+ }
+
+ public String checkValid(JsonArray snowrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (snowrow != null && snowrow.size() > 0) {
+ JsonElement element = snowrow.get(1);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer snowflakereturndatasb = new StringBuffer();
+
+ snowflakereturndatasb.append("{ \"statusCode\":");
+ snowflakereturndatasb.append(statusCode);
+ snowflakereturndatasb.append(",");
+ snowflakereturndatasb.append(" \"body\": {");
+ snowflakereturndatasb.append(" \"data\": [");
+ snowflakereturndatasb.append("] }}");
+ System.out.println("in exception with ");
+ return snowflakereturndatasb.toString();
+ }
+
+ public String formatReturnValue(int statusCode, JsonArray snowflakedata, boolean error, Cipher thalesCipher,
+ String datatype) throws IllegalBlockSizeException, BadPaddingException {
+ int row_number = 0;
+
+ String encdata = null;
+ String sensitive = null;
+
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+ JsonArray innerDataArray = new JsonArray();
+ int nbrofrows = snowflakedata.size();
+
+ for (int i = 0; i < nbrofrows; i++) {
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+ for (int j = 0; j < snowflakerow.size(); j++) {
+ if (j == 1) {
+ sensitive = checkValid(snowflakerow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ innerDataArray.add(sensitive);
+ // Can not return number since a leading 0 will not work.
+ // innerDataArray.add(new BigInteger(sensitive));
+ } else
+ innerDataArray.add(BADDATATAG);
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+ innerDataArray.add("");
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ innerDataArray.add(sensitive);
+ } else {
+ innerDataArray.add(sensitive);
+ }
+
+ } else {
+ if (!error) {
+ byte[] outbuf = thalesCipher.doFinal(sensitive.getBytes());
+ encdata = new String(outbuf);
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ innerDataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ innerDataArray.add(encdata);
+ }
+ } else {
+ encdata = sensitive;
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ innerDataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ innerDataArray.add(encdata);
+ }
+ }
+ }
+
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ row_number = snowflakecolumn.getAsInt();
+ innerDataArray.add(row_number);
+ }
+ }
+ }
+
+ bodyObject.add("data", dataArray);
+ JsonObject inputJsonObject = new JsonObject();
+ String bodyString = bodyObject.toString();
+ inputJsonObject.addProperty("statusCode", 200);
+ inputJsonObject.addProperty("body", bodyString);
+
+ String formattedStringnew = inputJsonObject.toString();
+
+ return formattedStringnew;
+ }
+
+ @Override
+ public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException {
+ // context.getLogger().log("Input: " + inputStream);
+ String input = IOUtils.toString(inputStream, "UTF-8");
+
+ Map encryptedErrorMapTotal = new HashMap();
+
+ String tweakAlgo = null;
+ String tweakData = null;
+ String snowflakereturnstring = null;
+ JsonObject body = null;
+ int statusCode = 200;
+ int numberofchunks = 0;
+
+ String callerStr = null;
+
+ // https://www.baeldung.com/java-aws-lambda
+
+ JsonObject snowflakeinput = null;
+
+ JsonArray snowflakedata = null;
+ NAESession session = null;
+ String keyName = "testfaas";
+ // String keyName = System.getenv("CMKEYNAME");
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be
+ // returned when the user above does not have access to the key and if doing a lookup in the userset
+ // and the user does not exist. If returnciphertextforuserwithnokeyaccess = no then an error will be
+ // returned to the query, else the results set will provide ciphertext.
+ // yes/no
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB?
+ // yes/no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it
+ // is the userset in CM but could be a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ // datatype - data type in db/actual data format/return type. valid values are char, charint, nbr, charintchar
+ String datatype = System.getenv("datatype");
+ // mode of operation valid values are : encrypt or decrypt
+ String mode = System.getenv("mode");
+
+ int batchsize = Integer.parseInt(System.getenv("BATCHSIZE"));
+
+ int cipherType = 0;
+ if (mode.equals("encrypt"))
+ cipherType = Cipher.ENCRYPT_MODE;
+ else
+ cipherType = Cipher.DECRYPT_MODE;
+
+ try {
+
+ JsonElement rootNode = JsonParser.parseString(input).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ snowflakeinput = rootNode.getAsJsonObject();
+ if (snowflakeinput.isJsonObject()) {
+ // For some reason when using snowflake it adds \n and \ to quotes in json.
+ // the JsonParser.parseString(input).getAsJsonObject(); is supposed to remove
+ // all of those
+ // characters but it does not do it for snowflake json.
+ JsonElement bodyele = snowflakeinput.get("body");
+ String bodystr = bodyele.getAsString().replaceAll(System.lineSeparator(), "");
+ bodystr = bodystr.replaceAll("\\\\", "");
+ // System.out.println("bodystr after replace" + bodystr);
+ body = gson.fromJson(bodystr, JsonObject.class);
+ snowflakedata = body.getAsJsonArray("data");
+
+ JsonObject requestContext = snowflakeinput.getAsJsonObject("requestContext");
+
+ if (requestContext != null) {
+ JsonObject identity = requestContext.getAsJsonObject("identity");
+
+ if (identity != null) {
+ callerStr = identity.get("user").getAsString();
+ System.out.println("user: " + callerStr);
+ } else {
+ System.out.println("Identity not found.");
+ }
+ } else {
+ System.out.println("Request context not found.");
+ }
+
+ if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group
+
+ boolean founduserinuserset = findUserInUserSet(callerStr, userName, password, usersetID,
+ userSetLookupIP);
+ // System.out.println("Found User " + founduserinuserset);
+ if (!founduserinuserset)
+ throw new CustomException("1001, User Not in User Set", 1001);
+
+ } else {
+ usersetlookupbool = false;
+ }
+
+ } else {
+ System.out.println("eerror");
+
+ }
+ }
+ System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
+ "CADP_for_JAVA.properties");
+ IngrianProvider builder = new Builder().addConfigFileInputStream(
+ getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
+
+ session = NAESession.getSession(userName, password.toCharArray());
+ NAEKey key = NAEKey.getSecretKey(keyName, session);
+ String algorithm = null;
+
+ if (datatype.equals("char"))
+ algorithm = "FPE/FF1/CARD62";
+ else if (datatype.equals("charint"))
+ algorithm = "FPE/FF1/CARD10";
+ else
+ algorithm = "FPE/FF1/CARD10";
+
+ int row_number = 0;
+
+ AbstractNAECipher thalesCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider");
+
+ System.out.println("Batchsize = " + batchsize);
+ int numberOfLines = snowflakedata.size();
+ int totalRowsLeft = numberOfLines;
+
+ if (batchsize > numberOfLines)
+ batchsize = numberOfLines;
+ if (batchsize >= BATCHLIMIT)
+ batchsize = BATCHLIMIT;
+
+ FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
+ .build();
+
+ spec = new FPEParameterAndFormatSpec[batchsize];
+ data = new byte[batchsize][];
+ snowrownbrs = new Integer[batchsize];
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+ JsonArray innerDataArray = new JsonArray();
+
+ thalesCipher.init(cipherType, key, spec[0]);
+
+ int i = 0;
+ int count = 0;
+ boolean newchunk = true;
+ int dataIndex = 0;
+ int specIndex = 0;
+ int snowRowIndex = 0;
+ String sensitive = null;
+
+ while (i < numberOfLines) {
+ int index = 0;
+
+ if (newchunk) {
+
+ if (totalRowsLeft < batchsize) {
+ spec = new FPEParameterAndFormatSpec[totalRowsLeft];
+ data = new byte[totalRowsLeft][];
+ snowrownbrs = new Integer[totalRowsLeft];
+ } else {
+ spec = new FPEParameterAndFormatSpec[batchsize];
+ data = new byte[batchsize][];
+ snowrownbrs = new Integer[batchsize];
+ }
+ newchunk = false;
+ }
+
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ for (int j = 0; j < snowflakerow.size(); j++) {
+
+ if (j == 1) {
+
+ // FPE example
+ sensitive = checkValid(snowflakerow);
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ sensitive = BADDATATAG + sensitive;
+ data[dataIndex++] = sensitive.getBytes();
+ // Can not return number since a leading 0 will not work.
+ // innerDataArray.add(new BigInteger(sensitive));
+ } else
+ data[dataIndex++] = BADDATATAG.getBytes();
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+ data[dataIndex++] = sensitive.getBytes();
+ } else {
+ data[dataIndex++] = sensitive.getBytes();
+ }
+ spec[specIndex++] = param;
+
+ } else {
+ data[dataIndex++] = sensitive.getBytes();
+ spec[specIndex++] = param;
+ }
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ row_number = snowflakecolumn.getAsInt();
+ snowrownbrs[snowRowIndex++] = row_number;
+
+ }
+
+ }
+
+ if (count == batchsize - 1) {
+
+ // Map to store exceptions while encryption
+ Map encryptedErrorMap = new HashMap();
+
+ byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap);
+
+ for (Map.Entry entry : encryptedErrorMap.entrySet()) {
+ Integer mkey = entry.getKey();
+ String mvalue = entry.getValue();
+ encryptedErrorMapTotal.put(mkey, mvalue);
+ }
+
+ for (int enc = 0; enc < encryptedData.length; enc++) {
+
+ innerDataArray.add(snowrownbrs[enc]);
+ innerDataArray.add(new String(encryptedData[enc]));
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ index++;
+
+ }
+
+ numberofchunks++;
+ newchunk = true;
+ count = 0;
+ dataIndex = 0;
+ specIndex = 0;
+ snowRowIndex = 0;
+ } else
+ count++;
+
+ totalRowsLeft--;
+ i++;
+ }
+
+ if (count > 0) {
+ numberofchunks++;
+ int index = 0;
+ Map encryptedErrorMap = new HashMap();
+ byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap);
+ for (int enc = 0; enc < encryptedData.length; enc++) {
+ innerDataArray.add(snowrownbrs[enc]);
+ innerDataArray.add(new String(encryptedData[enc]));
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ index++;
+
+ }
+ }
+
+ bodyObject.add("data", dataArray);
+ JsonObject inputJsonObject = new JsonObject();
+ String bodyString = bodyObject.toString();
+ inputJsonObject.addProperty("statusCode", 200);
+ inputJsonObject.addProperty("body", bodyString);
+
+ snowflakereturnstring = inputJsonObject.toString();
+ // System.out.println("snowflakereturnstring = " + snowflakereturnstring);
+
+ } catch (
+
+ Exception e) {
+ System.out.println("in exception with " + e.getMessage());
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
+
+ try {
+ snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, true, null, datatype);
+ } catch (IllegalBlockSizeException e1) {
+ // TODO Auto-generated catch block
+ e1.printStackTrace();
+ } catch (BadPaddingException e1) {
+ // TODO Auto-generated catch block
+ e1.printStackTrace();
+ }
+
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } finally {
+ if (session != null) {
+ session.closeSession();
+ }
+ }
+ System.out.println("number of chunks = " + numberofchunks);
+ outputStream.write(snowflakereturnstring.getBytes());
+ }
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCADPFPEUDF.java b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCADPFPEUDF.java
new file mode 100644
index 00000000..9f38b423
--- /dev/null
+++ b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCADPFPEUDF.java
@@ -0,0 +1,348 @@
+package example;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.math.BigInteger;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec;
+import com.ingrian.security.nae.IngrianProvider;
+import com.ingrian.security.nae.NAEKey;
+import com.ingrian.security.nae.NAESession;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
+import com.ingrian.security.nae.IngrianProvider.Builder;
+
+/* This sample AWS Lambda Function is used to implement a Snowflake Database User Defined Function(UDF). /*
+ * It is an example of how to use Thales Cipher Trust Application Data Protection (CADP)
+ * to protect sensitive data in a column. This example uses
+ * Format Preserve Encryption (FPE) to maintain the original format of the data
+ * so applications or business intelligence tools do not have to change in order
+ * to use these columns.
+ *
+ * Note: This source code is only to be used for testing and proof of concepts.
+ * Not production ready code. Was not tested for all possible data sizes and
+ * combinations of encryption algorithms and IV, etc. Was tested with CM 2.14 &
+ * CADP 8.15.0.001 For more information on CADP see link below.
+* For more information on CADP see link below.
+https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html
+* For more information on Snowflake External Functions see link below.
+https://docs.snowflake.com/en/sql-reference/external-functions-creating-aws
+ *
+ */
+
+public class ThalesAWSSnowCADPFPEUDF implements RequestStreamHandler {
+// private static final Logger logger = Logger.getLogger(LambdaRequestStreamHandlerCharEncryptFPE.class.getName());
+ private static final Gson gson = new Gson();
+ private static final String BADDATATAG = new String ("9999999999999999");
+ public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException {
+ // context.getLogger().log("Input: " + inputStream);
+ String input = IOUtils.toString(inputStream, "UTF-8");
+
+ String tweakAlgo = null;
+ String tweakData = null;
+ String snowflakereturnstring = null;
+ JsonObject body = null;
+ int statusCode = 200;
+
+ // https://www.baeldung.com/java-aws-lambda
+ JsonObject snowflakeinput = null;
+ String callerStr = null;
+ JsonArray snowflakedata = null;
+
+ NAESession session = null;
+
+ String keyName = "testfaas";
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be
+ // returned when the user above does not have access to the key and if doing a lookup in the userset
+ // and the user does not exist. If returnciphertextforuserwithnokeyaccess = no then an error will be
+ // returned to the query, else the results set will provide ciphertext.
+ // yes/no
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB?
+ // yes/no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it
+ // is the userset in CM but could be a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ //datatype - data type in db/actual data format/return type. valid values are char, charint, nbr, charintchar
+ String datatype = System.getenv("datatype");
+ //mode of operation valid values are : encrypt or decrypt
+ String mode = System.getenv("mode");
+
+ int cipherType = 0;
+ if (mode.equals("encrypt"))
+ cipherType = Cipher.ENCRYPT_MODE;
+ else
+ cipherType = Cipher.DECRYPT_MODE;
+
+
+ try {
+
+ JsonElement rootNode = JsonParser.parseString(input).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ snowflakeinput = rootNode.getAsJsonObject();
+ if (snowflakeinput.isJsonObject()) {
+ // For some reason when using snowflake it adds \n and \ to quotes in json.
+ // the JsonParser.parseString(input).getAsJsonObject(); is supposed to remove
+ // all of those
+ // characters but it does not do it for snowflake json.
+ JsonElement bodyele = snowflakeinput.get("body");
+ String bodystr = bodyele.getAsString().replaceAll(System.lineSeparator(), "");
+ // System.out.println("bodystr before replace" + bodystr );
+ bodystr = bodystr.replaceAll("\\\\", "");
+ // System.out.println("bodystr after replace" + bodystr );
+ body = gson.fromJson(bodystr, JsonObject.class);
+ snowflakedata = body.getAsJsonArray("data");
+ JsonObject requestContext = snowflakeinput.getAsJsonObject("requestContext");
+
+ if (requestContext != null) {
+ JsonObject identity = requestContext.getAsJsonObject("identity");
+
+ if (identity != null) {
+ callerStr = identity.get("user").getAsString();
+ System.out.println("user: " + callerStr);
+ } else {
+ System.out.println("Identity not found.");
+ }
+ } else {
+ System.out.println("Request context not found.");
+ }
+
+ if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group
+
+ boolean founduserinuserset = findUserInUserSet(callerStr, userName, password, usersetID,
+ userSetLookupIP);
+ // System.out.println("Found User " + founduserinuserset);
+ if (!founduserinuserset)
+ throw new CustomException("1001, User Not in User Set", 1001);
+
+ } else {
+ usersetlookupbool = false;
+ }
+
+ } else {
+ System.out.println("eerror");
+
+ }
+ }
+
+ // System.setProperty("com.ingrian.security.nae.NAE_IP.1", "10.20.1.9");
+ System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
+ "CADP_for_JAVA.properties");
+ IngrianProvider builder = new Builder().addConfigFileInputStream(
+ getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
+ session = NAESession.getSession(userName, password.toCharArray());
+ NAEKey key = NAEKey.getSecretKey(keyName, session);
+
+ String algorithm = null;
+
+ if (datatype.equals("char"))
+ algorithm = "FPE/FF1/CARD62";
+ else if (datatype.equals("charint"))
+ algorithm = "FPE/FF1/CARD10";
+ else
+ algorithm = "FPE/FF1/CARD10";
+
+ FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
+ .build();
+ ///
+ // ivSpec = param;
+ Cipher encryptCipher = Cipher.getInstance(algorithm, "IngrianProvider");
+
+ encryptCipher.init(cipherType, key, param);
+ snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, false, encryptCipher, datatype);
+
+ } catch (Exception e) {
+
+ System.out.println("in exception with " + e.getMessage());
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
+
+ try {
+ snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, true, null, datatype);
+ } catch (IllegalBlockSizeException e1) {
+ // TODO Auto-generated catch block
+ e1.printStackTrace();
+ } catch (BadPaddingException e1) {
+ // TODO Auto-generated catch block
+ e1.printStackTrace();
+ }
+
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+
+ } finally {
+ if (session != null) {
+ session.closeSession();
+ }
+ }
+
+ outputStream.write(snowflakereturnstring.getBytes());
+
+ }
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer snowflakereturndatasb = new StringBuffer();
+
+ snowflakereturndatasb.append("{ \"statusCode\":");
+ snowflakereturndatasb.append(statusCode);
+ snowflakereturndatasb.append(",");
+ snowflakereturndatasb.append(" \"body\": {");
+ snowflakereturndatasb.append(" \"data\": [");
+ snowflakereturndatasb.append("] }}");
+ System.out.println("in exception with ");
+ return snowflakereturndatasb.toString();
+ }
+
+ public String formatReturnValue(int statusCode, JsonArray snowflakedata, boolean error, Cipher thalesCipher,
+ String datatype) throws IllegalBlockSizeException, BadPaddingException {
+ int row_number = 0;
+
+ String encdata = null;
+ String sensitive = null;
+
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+ JsonArray innerDataArray = new JsonArray();
+ int nbrofrows = snowflakedata.size();
+
+ for (int i = 0; i < nbrofrows; i++) {
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+ for (int j = 0; j < snowflakerow.size(); j++) {
+ if (j == 1) {
+ sensitive = checkValid(snowflakerow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ innerDataArray.add(sensitive);
+ // Can not return number since a leading 0 will not work.
+ // innerDataArray.add(new BigInteger(sensitive));
+ } else
+ innerDataArray.add(BADDATATAG);
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+ innerDataArray.add("");
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ innerDataArray.add(sensitive);
+ } else {
+ innerDataArray.add(sensitive);
+ }
+
+ } else {
+ if (!error) {
+ byte[] outbuf = thalesCipher.doFinal(sensitive.getBytes());
+ encdata = new String(outbuf);
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ innerDataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ innerDataArray.add(encdata);
+ }
+ } else {
+ encdata = sensitive;
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ innerDataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ innerDataArray.add(encdata);
+ }
+ }
+ }
+
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ row_number = snowflakecolumn.getAsInt();
+ innerDataArray.add(row_number);
+ }
+ }
+ }
+
+ bodyObject.add("data", dataArray);
+ JsonObject inputJsonObject = new JsonObject();
+ String bodyString = bodyObject.toString();
+ inputJsonObject.addProperty("statusCode", 200);
+ inputJsonObject.addProperty("body", bodyString);
+
+ String formattedStringnew = inputJsonObject.toString();
+
+ return formattedStringnew;
+
+ }
+
+ public String checkValid(JsonArray snowrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (snowrow != null && snowrow.size() > 0) {
+ JsonElement element = snowrow.get(1);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+
+ boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
+
+ return founduserinuserset;
+
+ }
+
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCRDPFPEBulkUDF.java b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCRDPFPEBulkUDF.java
new file mode 100644
index 00000000..612ea040
--- /dev/null
+++ b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCRDPFPEBulkUDF.java
@@ -0,0 +1,595 @@
+package example;
+
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.math.BigInteger;
+import java.security.spec.AlgorithmParameterSpec;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.io.IOUtils;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+
+import okhttp3.MediaType;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
+/* This is a Thales CRDP UDF for Snowflake. It uses the Thales CRDP Bulk API.
+ * It is an example of how to use Thales CipherTrust REST Application Dataprotection (CRDP)
+ * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
+ * data so applications or business intelligence tools do not have to change in order to use these columns.
+*
+* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
+* for all possible data sizes and combinations of encryption algorithms and IV, etc.
+* Was tested with CM 2.14 and CRDP 1.0
+* For more information on CRDP see link below.
+https://thalesdocs.com/ctp/con/crdp/latest/index.html
+* For more information on Snowflake External Functions see link below.
+https://docs.snowflake.com/en/sql-reference/external-functions-creating-aws
+ *
+ *@author mwarner
+ *
+ */
+
+public class ThalesAWSSnowCRDPFPEBulkUDF implements RequestStreamHandler {
+
+ private static int BATCHLIMIT = 10000;
+ private static final String BADDATATAG = new String("9999999999999999");
+ private static final String REVEALRETURNTAG = new String("data");
+ private static final String PROTECTRETURNTAG = new String("protected_data");
+
+ private static final Gson gson = new Gson();
+
+ public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException {
+ // context.getLogger().log("Input: " + inputStream);
+ String input = IOUtils.toString(inputStream, "UTF-8");
+
+ Map snowErrorMap = new HashMap();
+ String encdata = "";
+ int error_count = 0;
+ int statusCode = 200;
+
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+ JsonArray innerDataArray = new JsonArray();
+ String snowflakereturnstring = null;
+ JsonObject body_input_request = null;
+
+ int numberofchunks = 0;
+
+ String callerStr = null;
+
+ JsonObject snowflakeinput = null;
+
+ JsonArray snowflakedata = null;
+
+ String keyName = "testfaas";
+ String crdpip = System.getenv("CRDPIP");
+ // String keyName = System.getenv("CMKEYNAME");
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned
+ // when the user above does not have access to the key and if doing a
+ // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no
+ // then an error will be returned to the query, else the results set will provide ciphertext.
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ // yes,no
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB
+ // yes,no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be
+ // a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ String keymetadatalocation = System.getenv("keymetadatalocation");
+ String external_version_from_ext_source = System.getenv("keymetadata");
+ String protection_profile = System.getenv("protection_profile");
+ String mode = System.getenv("mode");
+ String datatype = System.getenv("datatype");
+ int batchsize = Integer.parseInt(System.getenv("BATCHSIZE"));
+
+
+ String inputDataKey = null;
+ String outputDataKey = null;
+ String protectedData = null;
+ String externalkeymetadata = null;
+ String jsonBody = null;
+
+ String jsonTagForProtectReveal = null;
+
+ boolean bad_data = false;
+ String showrevealkey = "yes";
+
+ if (mode.equals("protectbulk")) {
+ inputDataKey = "data_array";
+ outputDataKey = "protected_data_array";
+ jsonTagForProtectReveal = PROTECTRETURNTAG;
+ if (keymetadatalocation.equalsIgnoreCase("internal")) {
+ showrevealkey = System.getenv("showrevealinternalkey");
+ if (showrevealkey == null)
+ showrevealkey = "yes";
+ }
+ } else {
+ inputDataKey = "protected_data_array";
+ outputDataKey = "data_array";
+ jsonTagForProtectReveal = REVEALRETURNTAG;
+ }
+
+ boolean showrevealkeybool = showrevealkey.equalsIgnoreCase("yes");
+
+ try {
+
+ JsonElement rootNode = JsonParser.parseString(input).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ snowflakeinput = rootNode.getAsJsonObject();
+ if (snowflakeinput.isJsonObject()) {
+ // For some reason when using snowflake it adds \n and \ to quotes in json.
+ // the JsonParser.parseString(input).getAsJsonObject(); is supposed to remove
+ // all of those
+ // characters but it does not do it for snowflake json.
+ JsonElement bodyele = snowflakeinput.get("body");
+ String bodystr = bodyele.getAsString().replaceAll(System.lineSeparator(), "");
+ bodystr = bodystr.replaceAll("\\\\", "");
+ // System.out.println("bodystr after replace" + bodystr);
+ body_input_request = gson.fromJson(bodystr, JsonObject.class);
+ snowflakedata = body_input_request.getAsJsonArray("data");
+
+ JsonObject requestContext = snowflakeinput.getAsJsonObject("requestContext");
+
+ if (requestContext != null) {
+ JsonObject identity = requestContext.getAsJsonObject("identity");
+
+ if (identity != null) {
+ callerStr = identity.get("user").getAsString();
+ System.out.println("user: " + callerStr);
+ } else {
+ System.out.println("Identity not found.");
+ }
+ } else {
+ System.out.println("Request context not found.");
+ }
+
+ if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group
+
+ boolean founduserinuserset = findUserInUserSet(callerStr, userName, password, usersetID,
+ userSetLookupIP);
+ // System.out.println("Found User " + founduserinuserset);
+ if (!founduserinuserset)
+ throw new CustomException("1001, User Not in User Set", 1001);
+
+ } else {
+ usersetlookupbool = false;
+ }
+
+ } else {
+ System.out.println("eerror");
+
+ }
+ }
+
+ StringBuffer protection_policy_buff = new StringBuffer();
+ String notvalid = "notvalid";
+
+ int numberOfLines = snowflakedata.size();
+ int totalRowsLeft = numberOfLines;
+
+ if (batchsize > numberOfLines)
+ batchsize = numberOfLines;
+ if (batchsize >= BATCHLIMIT)
+ batchsize = BATCHLIMIT;
+
+ int i = 0;
+ int count = 0;
+ int totalcount = 0;
+
+ int dataIndex = 0; // assumes index from snowflake will always be sequential.
+ JsonObject crdp_payload = new JsonObject();
+ String sensitive = null;
+ JsonArray crdp_payload_array = new JsonArray();
+
+ OkHttpClient client = new OkHttpClient().newBuilder().build();
+ MediaType mediaType = MediaType.parse("application/json");
+ String urlStr = "http://" + crdpip + ":8090/v1/" + mode;
+
+ while (i < numberOfLines) {
+
+ for (int b = 0; b < batchsize && b < totalRowsLeft; b++) {
+
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ sensitive = checkValid(snowflakerow);
+ protection_profile = protection_profile.trim();
+ // Format the output
+ String formattedElement = String.format("\"protection_policy_name\" : \"%s\"", protection_profile);
+ protection_policy_buff.append(formattedElement);
+ protection_policy_buff.append(",");
+
+ if (mode.equals("protectbulk")) {
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ // System.out.println("adding null not charint or nbr");
+ sensitive = sensitive.replace("notvalid", "");
+ sensitive = BADDATATAG + sensitive;
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ // sensitive = sensitive.replace("notvalid", "");
+
+ }
+ encdata = sensitive;
+
+ }
+ crdp_payload_array.add(sensitive);
+ } else {
+ JsonObject protectedDataObject = new JsonObject();
+ protectedDataObject.addProperty("protected_data", sensitive);
+ if (keymetadatalocation.equalsIgnoreCase("external")) {
+ protectedDataObject.addProperty("external_version", external_version_from_ext_source);
+ }
+ crdp_payload_array.add(protectedDataObject);
+
+ }
+
+ if (count == batchsize - 1) {
+ crdp_payload.add(inputDataKey, crdp_payload_array);
+ String inputdataarray = null;
+ if (mode.equals("revealbulk")) {
+ crdp_payload.addProperty("username", callerStr);
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ jsonBody = protection_policy_buff.toString();
+ jsonBody = jsonBody.replaceFirst("\\{", " ");
+
+ } else {
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ inputdataarray = protection_policy_buff.toString();
+ jsonBody = inputdataarray.replace("{", " ");
+ }
+ jsonBody = "{" + jsonBody;
+
+ // System.out.println(jsonBody);
+ RequestBody body = RequestBody.create(mediaType, jsonBody);
+
+ // System.out.println(urlStr);
+ Request crdp_request = new Request.Builder().url(urlStr).method("POST", body)
+ .addHeader("Content-Type", "application/json").build();
+ Response crdp_response = client.newCall(crdp_request).execute();
+
+ if (crdp_response.isSuccessful()) {
+ // Parse JSON response
+ String responseBody = crdp_response.body().string();
+ JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class);
+ JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey);
+
+ String status = jsonObject.get("status").getAsString();
+ int success_count = jsonObject.get("success_count").getAsInt();
+ error_count = jsonObject.get("error_count").getAsInt();
+ if (error_count > 0)
+ System.out.println("errors " + error_count);
+
+ for (JsonElement element : protectedDataArray) {
+
+ JsonObject protectedDataObject = element.getAsJsonObject();
+ if (protectedDataObject.has(jsonTagForProtectReveal)) {
+
+ protectedData = protectedDataObject.get(jsonTagForProtectReveal).getAsString();
+ // System.out.println(protectedData);
+ if (keymetadatalocation.equalsIgnoreCase("internal") && mode.equalsIgnoreCase("protectbulk") && !showrevealkeybool) {
+ if (protectedData.length()>7)
+ protectedData = protectedData.substring(7);
+ }
+
+ innerDataArray.add(dataIndex);
+ innerDataArray.add(new String(protectedData));
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+ if (mode.equals("protectbulk")) {
+ if (keymetadatalocation.equalsIgnoreCase("external")
+ && mode.equalsIgnoreCase("protectbulk")) {
+ externalkeymetadata = protectedDataObject.get("external_version")
+ .getAsString();
+ // System.out.println("Protected Data ext key metadata need to store this: "
+ // + externalkeymetadata);
+
+ }
+ }
+ } else if (protectedDataObject.has("error_message")) {
+ String errorMessage = protectedDataObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ snowErrorMap.put(i, errorMessage);
+ bad_data = true;
+ } else
+ System.out.println("unexpected json value from results: ");
+ dataIndex++;
+
+ }
+
+ crdp_payload_array = new JsonArray();
+ protection_policy_buff = new StringBuffer();
+ numberofchunks++;
+ totalcount = totalcount + count;
+ count = 0;
+ } else {// throw error....
+ System.err.println("Request failed with status code: " + crdp_response.code());
+ throw new CustomException("1010, Unexpected Error ", 1010);
+ }
+
+ } else {
+ count++;
+ }
+ totalRowsLeft--;
+ i++;
+ }
+ }
+ if (count > 0) {
+ crdp_payload.add(inputDataKey, crdp_payload_array);
+ String inputdataarray = null;
+ if (mode.equals("revealbulk")) {
+ crdp_payload.addProperty("username", callerStr);
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ jsonBody = protection_policy_buff.toString();
+ jsonBody = jsonBody.replaceFirst("\\{", " ");
+
+ } else {
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ inputdataarray = protection_policy_buff.toString();
+ jsonBody = inputdataarray.replace("{", " ");
+ }
+ jsonBody = "{" + jsonBody;
+
+ // System.out.println(jsonBody);
+ RequestBody body = RequestBody.create(mediaType, jsonBody);
+
+ // System.out.println(urlStr);
+ Request crdp_request = new Request.Builder().url(urlStr).method("POST", body)
+ .addHeader("Content-Type", "application/json").build();
+ Response crdp_response = client.newCall(crdp_request).execute();
+ String crdpreturnstr = null;
+ if (crdp_response.isSuccessful()) {
+ // Parse JSON response
+ String responseBody = crdp_response.body().string();
+ JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class);
+ JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey);
+
+ String status = jsonObject.get("status").getAsString();
+ int success_count = jsonObject.get("success_count").getAsInt();
+ error_count = jsonObject.get("error_count").getAsInt();
+ if (error_count > 0)
+ System.out.println("errors " + error_count);
+
+ if (mode.equals("protectbulk")) {
+
+ for (JsonElement element : protectedDataArray) {
+ JsonObject protectedDataObject = element.getAsJsonObject();
+ if (protectedDataObject.has("protected_data")) {
+
+ protectedData = protectedDataObject.get("protected_data").getAsString();
+
+ if (keymetadatalocation.equalsIgnoreCase("internal") && mode.equalsIgnoreCase("protectbulk") && !showrevealkeybool) {
+ if (protectedData.length()>7)
+ protectedData = protectedData.substring(7);
+ }
+
+ innerDataArray.add(dataIndex);
+ innerDataArray.add(new String(protectedData));
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ if (keymetadatalocation.equalsIgnoreCase("external")
+ && mode.equalsIgnoreCase("protectbulk")) {
+ externalkeymetadata = protectedDataObject.get("external_version").getAsString();
+ // System.out.println("Protected Data ext key metadata need to store this: "
+ // + externalkeymetadata);
+
+ }
+ } else if (protectedDataObject.has("error_message")) {
+ String errorMessage = protectedDataObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ snowErrorMap.put(i, errorMessage);
+ bad_data = true;
+ } else {
+ System.out.println("unexpected json value from results: ");
+ throw new CustomException("1010, Unexpected Error ", 1010);
+ }
+ dataIndex++;
+ }
+ } else {
+ // reveal logic
+
+ for (JsonElement element : protectedDataArray) {
+ JsonObject protectedDataObject = element.getAsJsonObject();
+ if (protectedDataObject.has("data")) {
+ protectedData = protectedDataObject.get("data").getAsString();
+ // System.out.println(protectedData);
+
+ innerDataArray.add(dataIndex);
+ innerDataArray.add(new String(protectedData));
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ } else if (protectedDataObject.has("error_message")) {
+ String errorMessage = protectedDataObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ snowErrorMap.put(i, errorMessage);
+ bad_data = true;
+ } else
+ System.out.println("unexpected json value from results: ");
+ dataIndex++;
+ }
+ }
+
+ crdp_response.close();
+
+ numberofchunks++;
+
+ totalcount = totalcount + count;
+ count = 0;
+
+ } else {
+ System.err.println("Request failed with status code: " + crdp_response.code());
+ }
+ }
+ System.out.println("total chuncks " + numberofchunks);
+
+ bodyObject.add("data", dataArray);
+ JsonObject inputJsonObject = new JsonObject();
+ String bodyString = bodyObject.toString();
+ inputJsonObject.addProperty("statusCode", 200);
+ inputJsonObject.addProperty("body", bodyString);
+
+ snowflakereturnstring = inputJsonObject.toString();
+
+ } catch (Exception e) {
+ System.out.println("in exception with " + e.getMessage());
+ snowflakereturnstring = "exception ";
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
+
+ bodyObject = new JsonObject();
+ dataArray = new JsonArray();
+ innerDataArray = new JsonArray();
+ int nbrofrows = snowflakedata.size();
+ for (int i = 0; i < nbrofrows; i++) {
+
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ for (int j = 0; j < snowflakerow.size(); j++) {
+ if (j == 1) {
+ // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
+ // FPE example
+ String sensitive = checkValid(snowflakerow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null")
+ || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+
+ }
+ encdata = sensitive;
+
+ } else {
+ // System.out.println("normal number data" + sensitive);
+ }
+ innerDataArray.add(sensitive);
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ int row_number = snowflakecolumn.getAsInt();
+ innerDataArray.add(row_number);
+ }
+ }
+ }
+
+ bodyObject.add("data", dataArray);
+ JsonObject inputJsonObject = new JsonObject();
+ String bodyString = bodyObject.toString();
+ inputJsonObject.addProperty("statusCode", 200);
+ inputJsonObject.addProperty("body", bodyString);
+
+ snowflakereturnstring = inputJsonObject.toString();
+ // System.out.println(" new data " + snowflakereturnstring);
+
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+
+ } finally {
+
+ }
+ // System.out.println(snowflakereturnstring);
+ outputStream.write(snowflakereturnstring.getBytes());
+
+ }
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+
+ boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
+
+ return founduserinuserset;
+
+ }
+
+ public String checkValid(JsonArray snowrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (snowrow != null && snowrow.size() > 0) {
+ JsonElement element = snowrow.get(1);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer snowflakereturndatasb = new StringBuffer();
+
+ snowflakereturndatasb.append("{ \"statusCode\":");
+ snowflakereturndatasb.append(statusCode);
+ snowflakereturndatasb.append(",");
+ snowflakereturndatasb.append(" \"body\": {");
+ snowflakereturndatasb.append(" \"data\": [");
+ snowflakereturndatasb.append("] }}");
+ System.out.println("in exception with ");
+ return snowflakereturndatasb.toString();
+ }
+
+
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCRDPFPEUDF.java b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCRDPFPEUDF.java
new file mode 100644
index 00000000..392395f8
--- /dev/null
+++ b/database/snowflake/Thales-Snow-AWS-UDF/src/main/java/example/ThalesAWSSnowCRDPFPEUDF.java
@@ -0,0 +1,447 @@
+package example;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.io.IOUtils;
+
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+
+import okhttp3.MediaType;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
+/* This is a Thales CRDP UDF for Snowflake.
+ * It is an example of how to use Thales CipherTrust REST Application Dataprotection (CRDP)
+ * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
+ * data so applications or business intelligence tools do not have to change in order to use these columns.
+*
+* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
+* for all possible data sizes and combinations of encryption algorithms and IV, etc.
+* Was tested with CM 2.14 and CRDP 1.0
+* For more information on CRDP see link below.
+https://thalesdocs.com/ctp/con/crdp/latest/index.html
+* For more information on Snowflake External Functions see link below.
+https://docs.snowflake.com/en/sql-reference/external-functions-creating-aws
+ *
+ *@author mwarner
+ *
+ */
+
+public class ThalesAWSSnowCRDPFPEUDF implements RequestStreamHandler {
+
+ private static final Gson gson = new Gson();
+ private static final String BADDATATAG = new String ("9999999999999999");
+ private static final String REVEALRETURNTAG = new String("data");
+ private static final String PROTECTRETURNTAG = new String("protected_data");
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer snowflakereturndatasb = new StringBuffer();
+
+ snowflakereturndatasb.append("{ \"statusCode\":");
+ snowflakereturndatasb.append(statusCode);
+ snowflakereturndatasb.append(",");
+ snowflakereturndatasb.append(" \"body\": {");
+ snowflakereturndatasb.append(" \"data\": [");
+ snowflakereturndatasb.append("] }}");
+ System.out.println("in exception with ");
+ return snowflakereturndatasb.toString();
+ }
+
+ public String checkValid(JsonArray snowrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (snowrow != null && snowrow.size() > 0) {
+ JsonElement element = snowrow.get(1);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ @Override
+ public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException {
+ Map snowErrroMap = new HashMap();
+ String encdata = "";
+ String snowflakereturnstring = null;
+ String input = IOUtils.toString(inputStream, "UTF-8");
+ //String input = inputStream;
+ JsonObject snowflakebody = null;
+ int statusCode = 200;
+
+ // https://www.baeldung.com/java-aws-lambda
+ JsonObject snowflakeinput = null;
+ String callerStr = null;
+ JsonArray snowflakedata = null;
+
+ String keyName = "testfaas";
+ String crdpip = System.getenv("CRDPIP");
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be
+ // returned when the user above does not have access to the key and if doing a lookup in the userset
+ // and the user does not exist. If returnciphertextforuserwithnokeyaccess = no then an error will be
+ // returned to the query, else the results set will provide ciphertext.
+ // yes/no
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB?
+ // yes/no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it
+ // is the userset in CM but could be a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ // keymetadatalocation keymeta data can be internal or external
+ String keymetadatalocation = System.getenv("keymetadatalocation");
+ // keymetadata represents a 7 digit value that contains policy version and key version. example 1001001.
+ // Normally would come from a database
+ String external_version_from_ext_source = System.getenv("keymetadata");
+ // protection_profile = the protection profile to be used for protect or reveal. This is in CM under application
+ // data protection/protection profiles.
+ String protection_profile = System.getenv("protection_profile");
+ // mode of operation. valid values are protect/reveal
+ String mode = System.getenv("mode");
+ String datatype = System.getenv("datatype");
+
+ String dataKey = null;
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+ JsonArray innerDataArray = new JsonArray();
+ String jsonTagForProtectReveal = null;
+
+ boolean bad_data = false;
+ String showrevealkey = "yes";
+
+ if (mode.equals("protect")) {
+ dataKey = "data";
+ jsonTagForProtectReveal = PROTECTRETURNTAG;
+ if (keymetadatalocation.equalsIgnoreCase("internal")) {
+ showrevealkey = System.getenv("showrevealinternalkey");
+ if (showrevealkey == null)
+ showrevealkey = "yes";
+ }
+ } else {
+ dataKey = "protected_data";
+ jsonTagForProtectReveal = REVEALRETURNTAG;
+ }
+
+ boolean showrevealkeybool = showrevealkey.equalsIgnoreCase("yes");
+
+ // This code is only to be used when input data contains user info.
+
+ try {
+
+ JsonElement rootNode = JsonParser.parseString(input).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ snowflakeinput = rootNode.getAsJsonObject();
+ if (snowflakeinput.isJsonObject()) {
+ // For some reason when using snowflake it adds \n and \ to quotes in json.
+ // the JsonParser.parseString(input).getAsJsonObject(); is supposed to remove
+ // all of those
+ // characters but it does not do it for snowflake json.
+ JsonElement bodyele = snowflakeinput.get("body");
+ String bodystr = bodyele.getAsString().replaceAll(System.lineSeparator(), "");
+ // System.out.println("bodystr before replace" + bodystr );
+ bodystr = bodystr.replaceAll("\\\\", "");
+ // System.out.println("bodystr after replace" + bodystr );
+ snowflakebody = gson.fromJson(bodystr, JsonObject.class);
+ snowflakedata = snowflakebody.getAsJsonArray("data");
+ JsonObject requestContext = snowflakeinput.getAsJsonObject("requestContext");
+
+ if (requestContext != null) {
+ JsonObject identity = requestContext.getAsJsonObject("identity");
+
+ if (identity != null) {
+ callerStr = identity.get("user").getAsString();
+ //String[] parts = callerStr.split(":");
+ //callerStr = parts[0];
+ System.out.println("user: " + callerStr);
+ } else {
+ System.out.println("Identity not found.");
+ }
+ } else {
+ System.out.println("Request context not found.");
+ }
+
+ if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group
+
+ boolean founduserinuserset = findUserInUserSet(callerStr, userName, password, usersetID,
+ userSetLookupIP);
+ // System.out.println("Found User " + founduserinuserset);
+ if (!founduserinuserset)
+ throw new CustomException("1001, User Not in User Set", 1001);
+
+ } else {
+ usersetlookupbool = false;
+ }
+
+ } else {
+ System.out.println("eerror");
+
+ }
+ }
+
+ // Serialization
+
+ String protectedData = null;
+ String externalkeymetadata = null;
+ String crdpjsonBody = null;
+ // String external_version_from_ext_source = "1004001";
+ // String external_version_from_ext_source = "1001001";
+ int row_number = 0;
+ JsonObject crdp_payload = new JsonObject();
+
+ crdp_payload.addProperty("protection_policy_name", protection_profile);
+ String sensitive = null;
+
+ OkHttpClient client = new OkHttpClient().newBuilder().build();
+ MediaType mediaType = MediaType.parse("application/json");
+ String urlStr = "http://" + crdpip + ":8090/v1/" + mode;
+ int nbrofrows = snowflakedata.size();
+
+ for (int i = 0; i < nbrofrows; i++) {
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ for (int j = 0; j < snowflakerow.size(); j++) {
+
+ if (j == 1) {
+ // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
+ // FPE example
+ sensitive = checkValid(snowflakerow);
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null")
+ || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+
+ }
+ encdata = sensitive;
+
+ } else {
+
+ crdp_payload.addProperty(dataKey, sensitive);
+ if (mode.equals("reveal")) {
+ crdp_payload.addProperty("username", callerStr);
+ if (keymetadatalocation.equalsIgnoreCase("external")) {
+ crdp_payload.addProperty("external_version", external_version_from_ext_source);
+ }
+ }
+ crdpjsonBody = crdp_payload.toString();
+ System.out.println(crdpjsonBody);
+ RequestBody body = RequestBody.create(mediaType, crdpjsonBody);
+
+ // String urlStr = "\"http://" + cmip + ":8090/v1/" + mode+ "\"";
+ System.out.println(urlStr);
+ Request crdp_request = new Request.Builder()
+ .url(urlStr).method("POST", body).addHeader("Content-Type", "application/json")
+ .build();
+ Response crdp_response = client.newCall(crdp_request).execute();
+
+ if (crdp_response.isSuccessful()) {
+
+ // Parse JSON response
+ String responseBody = crdp_response.body().string();
+ Gson gson = new Gson();
+ JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class);
+
+ if (mode.equals("protect")) {
+ if (jsonObject.has("protected_data")) {
+ protectedData = jsonObject.get("protected_data").getAsString();
+ if (keymetadatalocation.equalsIgnoreCase("external") && mode.equalsIgnoreCase("protect")) {
+ externalkeymetadata = jsonObject.get("external_version").getAsString();
+ // System.out.println("Protected Data ext key metadata need to store this: "
+ // + externalkeymetadata);
+ }
+
+ if (keymetadatalocation.equalsIgnoreCase("internal") && mode.equalsIgnoreCase("protect") && !showrevealkeybool) {
+ if (protectedData.length()>7)
+ protectedData = protectedData.substring(7);
+ }
+
+ } else if (jsonObject.has("error_message")) {
+ String errorMessage = jsonObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ snowErrroMap.put(i, errorMessage);
+ bad_data = true;
+ } else
+ System.out.println("unexpected json value from results: ");
+
+ }
+
+ else if (jsonObject.has("data")) {
+ protectedData = jsonObject.get("data").getAsString();
+ //System.out.println("Protected Data: " + protectedData);
+ } else if (jsonObject.has("error_message")) {
+ String errorMessage = jsonObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ snowErrroMap.put(i, errorMessage);
+ bad_data = true;
+ } else
+ System.out.println("unexpected json value from results: ");
+ } else {
+ System.err.println("Request failed with status code: " + crdp_response.code());
+ }
+
+ crdp_response.close();
+
+ encdata = protectedData;
+
+ }
+
+ innerDataArray.add(encdata);
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ row_number = snowflakecolumn.getAsInt();
+ innerDataArray.add(row_number);
+ }
+
+ }
+
+ }
+
+ bodyObject.add("data", dataArray);
+ JsonObject inputJsonObject = new JsonObject();
+ String bodyString = bodyObject.toString();
+ inputJsonObject.addProperty("statusCode", 200);
+ inputJsonObject.addProperty("body", bodyString);
+
+ snowflakereturnstring = inputJsonObject.toString();
+
+ } catch (Exception e) {
+
+ System.out.println("in exception with " + e.getMessage());
+ snowflakereturnstring = "exception ";
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
+
+ bodyObject = new JsonObject();
+ dataArray = new JsonArray();
+ innerDataArray = new JsonArray();
+ int nbrofrows = snowflakedata.size();
+ for (int i = 0; i < nbrofrows; i++) {
+
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ for (int j = 0; j < snowflakerow.size(); j++) {
+ if (j == 1) {
+ // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
+ // FPE example
+ String sensitive = checkValid(snowflakerow);
+
+ sensitive = checkValid(snowflakerow);
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ //System.out.println("adding null not charint or nbr");
+ sensitive = sensitive.replace("notvalid", "");
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null")
+ || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+
+ }
+ encdata = sensitive;
+
+ } else {
+ //System.out.println("normal number data" + sensitive);
+ }
+ innerDataArray.add(sensitive);
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ int row_number = snowflakecolumn.getAsInt();
+ innerDataArray.add(row_number);
+ }
+ }
+ }
+
+ bodyObject.add("data", dataArray);
+ JsonObject inputJsonObject = new JsonObject();
+ String bodyString = bodyObject.toString();
+ inputJsonObject.addProperty("statusCode", 200);
+ inputJsonObject.addProperty("body", bodyString);
+
+ snowflakereturnstring = inputJsonObject.toString();
+
+
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+
+ } finally {
+
+ }
+ //System.out.println("results" + snowflakereturnstring);
+ outputStream.write(snowflakereturnstring.getBytes());
+
+ }
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+
+ boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
+
+ return founduserinuserset;
+
+ }
+
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/CMUserSetHelper.java b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/CMUserSetHelper.java
new file mode 100644
index 00000000..06f17015
--- /dev/null
+++ b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/CMUserSetHelper.java
@@ -0,0 +1,503 @@
+package com.example;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.security.KeyStore;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509TrustManager;
+/*
+ * This app provides a number of different helper methods dealing with CM Application Data Protection UserSets. There is a method to find
+ * a user in a userset and another method to populate the userset from a flat file. Usersets are typically used within
+ * an access policy but they are not restricted to that usage.
+ * Was tested with CM 2.14
+ * Note: This source code is only to be used for testing and proof of concepts.
+ * @author mwarner
+ *
+ */
+
+public class CMUserSetHelper {
+
+ static String hostnamevalidate = "yourhostname";
+
+ String usersetid = "716f01a6-5cab-4799-925a-6dc2d8712fc1";
+ String cmIP = "yourip";
+ String apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?name=";
+ // + username_in_userset;
+ String addusertouserset = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users";
+
+ int totalrecords = 0;
+ String authUrl = "https://" + cmIP + "/api/v1/auth/tokens";
+
+ static boolean debug = true;
+ int chunksize = 5;
+ static int CHUNKSIZEMAX = 100;
+
+ public CMUserSetHelper(String usersetid, String cmIP) {
+
+ this.usersetid = usersetid;
+ this.cmIP = cmIP;
+ this.apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?name=";
+ this.addusertouserset = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users";
+ this.authUrl = "https://" + cmIP + "/api/v1/auth/tokens";
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ String username = args[0];
+ String password = args[1];
+ String cmip = args[2];
+ String usersetid = args[3];
+ String filePath = args[4];
+ CMUserSetHelper cmusersetHelper = new CMUserSetHelper(usersetid, cmip);
+ int totalrecords = 0;
+
+ String jwthtoken = geAuthToken(cmusersetHelper.authUrl, username, password);
+
+ String newtoken = "Bearer " + removeQuotes(jwthtoken);
+
+ // String filePath =
+ // "C:\\Users\\t0185905\\workspace\\CT-VL-GCP\\src\\main\\java\\com\\example\\emailAddresses.txt";
+ RandomAccessFile file = new RandomAccessFile(filePath, "r");
+ if (cmusersetHelper.chunksize > CHUNKSIZEMAX)
+ cmusersetHelper.chunksize = CHUNKSIZEMAX;
+ int totoalnbrofrecords = numberOfLines(file);
+ if (cmusersetHelper.chunksize > totoalnbrofrecords) {
+ cmusersetHelper.chunksize = totoalnbrofrecords / 2;
+ }
+ //totalrecords = cmusersetHelper.addAUserToUserSet(cmusersetHelper.addusertouserset, newtoken);
+ totalrecords = cmusersetHelper.addAUserToUserSetFromFile(cmusersetHelper.addusertouserset, newtoken, filePath);
+ System.out.println("Totalrecords inserted into Userset " + cmusersetHelper.usersetid + " = " + totalrecords);
+
+ }
+
+ /**
+ * Returns an boolean if user found
+ *
+ *
+ * @param user user to find
+ * @param newtoken jwt token to use
+ * @return boolean true if found in userset
+ * @throws CustomException
+ */
+ public boolean findUserInUserSet(String user, String newtoken) throws CustomException {
+ boolean found = false;
+
+ String apiUrl = this.apiUrlGetUsers + user;
+
+ apiUrl = removeQuotes(apiUrl);
+
+ try {
+ URL url = new URL(apiUrl);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestMethod("GET");
+ connection.setRequestProperty("Authorization", newtoken);
+ connection.setRequestProperty("accept", "application/json");
+
+ connection.setDoInput(true);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+
+ StringBuilder response = new StringBuilder();
+ String line;
+ while ((line = reader.readLine()) != null) {
+ response.append(line);
+ }
+ reader.close();
+
+ Gson gson = new Gson();
+ if (debug)
+ System.out.println("response " + response);
+ JsonObject input = null;
+ JsonElement total = null;
+ JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ input = rootNode.getAsJsonObject();
+ if (input.isJsonObject()) {
+ total = input.get("total");
+ }
+ }
+ JsonPrimitive column = total.getAsJsonPrimitive();
+ String totalstr = column.getAsJsonPrimitive().toString();
+
+ Integer i = Integer.valueOf(totalstr);
+
+ if (i > 0) {
+ found = true;
+
+ }
+ connection.disconnect();
+ } catch (Exception e) {
+ if (e.getMessage().contains("403")) {
+ throw new CustomException("1002, User Not in Application Data Protection Clients ", 1002);
+ } else
+ e.printStackTrace();
+ }
+
+ return found;
+
+ }
+
+ /**
+ * Loads users from a file to the userset. Returns an int of number of users
+ * added.
+ *
+ *
+ * @param url url to userset api
+ * @param newtoken jwt token to use
+ * @param filePath file to load
+ * @return int totalnumberofrecords added to userset
+ */
+
+ public int addAUserToUserSetFromFile(String url, String newtoken, String filePath) throws IOException {
+
+ int totalnbrofrecords = 0;
+ try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
+ String line;
+ int count = 0;
+ StringBuilder payloadBuilder = new StringBuilder();
+
+ payloadBuilder.append("{\"users\": [");
+
+ while ((line = br.readLine()) != null) {
+ totalnbrofrecords++;
+ payloadBuilder.append("\"").append(line).append("\",");
+ count++;
+
+ // If 'n' records have been read, print the payload
+ if (count == chunksize) {
+ makeCMCall(payloadBuilder, newtoken, url);
+ // Reset payload builder and count for the next batch of records
+ payloadBuilder = new StringBuilder("{\"users\": [");
+ count = 0;
+ }
+ }
+
+ // If there are remaining records, print the payload
+ if (count > 0) {
+ payloadBuilder.deleteCharAt(payloadBuilder.length() - 1); // Remove the trailing comma
+ payloadBuilder.append("}");
+ makeCMCall(payloadBuilder, newtoken, url);
+ if (debug)
+ System.out.println(payloadBuilder.toString());
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ return totalnbrofrecords;
+
+ }
+
+ /**
+ * Loads users from a file to the userset. Returns an int of number of users
+ * added.
+ *
+ *
+ * @param payloadBuilder payload for CM call that contains users to add
+ * @param newtoken jwt token to use
+ * @param url url to peform inserts
+ * @return int response code
+ */
+
+ public static int makeCMCall(StringBuilder payloadBuilder, String newtoken, String url) throws IOException {
+
+ payloadBuilder.deleteCharAt(payloadBuilder.length() - 1); // Remove the trailing comma
+ payloadBuilder.append("]}");
+ if (debug)
+ System.out.println(payloadBuilder.toString());
+ String payload = payloadBuilder.toString();
+
+ // Create URL object
+ URL apiUrl = new URL(url);
+ // Create connection object
+ HttpURLConnection connection = (HttpURLConnection) apiUrl.openConnection();
+ connection.setRequestMethod("POST");
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setDoOutput(true);
+ connection.setRequestProperty("Authorization", newtoken);
+ // Send request
+ OutputStream outputStream = connection.getOutputStream();
+ outputStream.write(payload.getBytes());
+ outputStream.flush();
+ outputStream.close();
+
+ // Get response
+ int responseCode = connection.getResponseCode();
+ BufferedReader reader;
+ if (responseCode >= 200 && responseCode < 300) {
+ reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+ } else {
+ reader = new BufferedReader(new InputStreamReader(connection.getErrorStream()));
+ }
+
+ // Read response
+ String lineresponse;
+ StringBuilder response = new StringBuilder();
+ while ((lineresponse = reader.readLine()) != null) {
+ response.append(lineresponse);
+ }
+ reader.close();
+
+ // Print response
+ if (debug) {
+ System.out.println("Response Code: " + responseCode);
+ System.out.println("Response Body: " + response.toString());
+ }
+ // Disconnect connection
+ connection.disconnect();
+ // Reset payload builder and count for the next batch of records
+ payloadBuilder = new StringBuilder("{\"users\": [");
+
+ return responseCode;
+
+ }
+
+ /**
+ * Simple sample of showing how to load a couple of users to the userset.
+ * Returns an int of number of users added.
+ *
+ *
+ * @param url url to userset api
+ * @param newtoken jwt token to use
+ *
+ * @return int response code
+ */
+ public static int addAUserToUserSet(String url, String newtoken) throws IOException {
+ boolean found = false;
+
+ String payload = "{\"users\": [\"akhip@company.com\",\"user2@company.com\"]}";
+
+ // Create URL object
+ URL apiUrl = new URL(url);
+ // Create connection object
+ HttpURLConnection connection = (HttpURLConnection) apiUrl.openConnection();
+ connection.setRequestMethod("POST");
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setDoOutput(true);
+ connection.setRequestProperty("Authorization", newtoken);
+ // Send request
+ OutputStream outputStream = connection.getOutputStream();
+ outputStream.write(payload.getBytes());
+ outputStream.flush();
+ outputStream.close();
+
+ // Get response
+ int responseCode = connection.getResponseCode();
+ BufferedReader reader;
+ if (responseCode >= 200 && responseCode < 300) {
+ reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+ } else {
+ reader = new BufferedReader(new InputStreamReader(connection.getErrorStream()));
+ }
+
+ // Read response
+ String line;
+ StringBuilder response = new StringBuilder();
+ while ((line = reader.readLine()) != null) {
+ response.append(line);
+ }
+ reader.close();
+
+ // Print response
+ if (debug) {
+ System.out.println("Response Code: " + responseCode);
+ System.out.println("Response Body: " + response.toString());
+ }
+ // Disconnect connection
+ connection.disconnect();
+
+ return responseCode;
+
+ }
+
+ public static String removeQuotes(String input) {
+
+ // Remove double quotes from the input string
+ input = input.replace("\"", "");
+
+ return input;
+ }
+
+ private static int numberOfLines(RandomAccessFile file) throws IOException {
+ int numberOfLines = 0;
+ while (file.readLine() != null) {
+ numberOfLines++;
+ }
+ file.seek(0);
+ return numberOfLines;
+ }
+
+ /**
+ * Get JWT from CM
+ *
+ *
+ * @param apiUrl url to CM auth
+ * @param username username on CM
+ * @param pwd password on CM
+ * @return String jwt
+ */
+
+ public static String geAuthToken(String apiUrl, String usernb, String pwd) throws Exception
+
+ {
+
+ String jStr = "{\"username\":\"" + usernb + "\",\"password\":\"" + pwd + "\"}";
+ disableCertValidation();
+
+ String jwtstr = null;
+ try {
+ URL url = new URL(apiUrl);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestProperty("Content-length", String.valueOf(jStr.length()));
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setRequestMethod("GET");
+ connection.setDoOutput(true);
+ connection.setDoInput(true);
+ DataOutputStream output = new DataOutputStream(connection.getOutputStream());
+ output.writeBytes(jStr);
+ output.close();
+ BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+ StringBuilder response = new StringBuilder();
+ String line;
+ while ((line = reader.readLine()) != null) {
+ response.append(line);
+ }
+ reader.close();
+
+ JsonObject input = null;
+ JsonElement jwt = null;
+ JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ input = rootNode.getAsJsonObject();
+ if (input.isJsonObject()) {
+ jwt = input.get("jwt");
+ }
+ }
+ JsonPrimitive column = jwt.getAsJsonPrimitive();
+ jwtstr = column.getAsJsonPrimitive().toString();
+ connection.disconnect();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ return jwtstr;
+
+ }
+
+ /**
+ * Get JWT from CM
+ *
+ *
+ * @param apiUrl url to CM auth
+ * @param username username on CM
+ * @param pwd password on CM
+ * @return String jwt
+ */
+
+ public static String geAuthToken(String apiUrl) throws Exception
+
+ {
+
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+ String jStr = "{\"username\":\"" + userName + "\",\"password\":\"" + password + "\"}";
+
+ disableCertValidation();
+
+ String jwtstr = null;
+ try {
+ URL url = new URL(apiUrl);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestProperty("Content-length", String.valueOf(jStr.length()));
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setRequestMethod("GET");
+ connection.setDoOutput(true);
+ connection.setDoInput(true);
+ DataOutputStream output = new DataOutputStream(connection.getOutputStream());
+ output.writeBytes(jStr);
+ output.close();
+ BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+ StringBuilder response = new StringBuilder();
+ String line;
+ while ((line = reader.readLine()) != null) {
+ response.append(line);
+ }
+ reader.close();
+ if (debug)
+ System.out.println("response " + response);
+ JsonObject input = null;
+ JsonElement jwt = null;
+ JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ input = rootNode.getAsJsonObject();
+ if (input.isJsonObject()) {
+ jwt = input.get("jwt");
+ }
+ }
+ JsonPrimitive column = jwt.getAsJsonPrimitive();
+ jwtstr = column.getAsJsonPrimitive().toString();
+ connection.disconnect();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ return jwtstr;
+
+ }
+
+ public static void disableCertValidation() throws Exception {
+ TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() {
+ public java.security.cert.X509Certificate[] getAcceptedIssuers() {
+ return null;
+ }
+
+ public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) {
+ }
+
+ public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) {
+ }
+ } };
+
+ // Install the all-trusting trust manager
+ try {
+ SSLContext sc = SSLContext.getInstance("SSL");
+ sc.init(null, trustAllCerts, new java.security.SecureRandom());
+ HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ // Create all-trusting host name verifier
+ HostnameVerifier allHostsValid = new HostnameVerifier() {
+ public boolean verify(String hostname, SSLSession session) {
+ return true;
+ }
+ };
+
+ // Install the all-trusting host verifier
+ HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid);
+ }
+
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/CustomException.java b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/CustomException.java
new file mode 100644
index 00000000..44796974
--- /dev/null
+++ b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/CustomException.java
@@ -0,0 +1,15 @@
+
+
+
+class CustomException extends Exception {
+ private int errorCode;
+
+ public CustomException(String message, int errorCode) {
+ super(message);
+ this.errorCode = errorCode;
+ }
+
+ public int getErrorCode() {
+ return errorCode;
+ }
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCADPFPEBulkUDFTester.java b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCADPFPEBulkUDFTester.java
new file mode 100644
index 00000000..2a0be92d
--- /dev/null
+++ b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCADPFPEBulkUDFTester.java
@@ -0,0 +1,612 @@
+
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.math.BigInteger;
+import java.security.spec.AlgorithmParameterSpec;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+
+import org.apache.commons.io.IOUtils;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+import com.ingrian.security.nae.AbstractNAECipher;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec;
+import com.ingrian.security.nae.IngrianProvider;
+import com.ingrian.security.nae.NAECipher;
+import com.ingrian.security.nae.NAEKey;
+import com.ingrian.security.nae.NAESession;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
+import com.ingrian.security.nae.IngrianProvider.Builder;
+
+/* This sample AWS Lambda Function is used to implement a Snowflake Database User Defined Function(UDF). /*
+ * It is an example of how to use Thales Cipher Trust Application Data Protection (CADP)
+ * to protect sensitive data in a column. This example uses
+ * Format Preserve Encryption (FPE) to maintain the original format of the data
+ * so applications or business intelligence tools do not have to change in order
+ * to use these columns.
+*
+* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
+* for all possible data sizes and combinations of encryption algorithms and IV, etc.
+* Was tested with CM 2.14 & CADP CADP 8.15.0.001
+* For more information on CADP see link below.
+https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html
+* For more information on Snowflake External Functions see link below.
+https://docs.snowflake.com/en/sql-reference/external-functions-creating-aws
+
+Notes: This example uses the CADP bulk API.
+Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data
+element has Unicode characters then the supported size limit would be 1750 characters
+
+Size of spec array should be same as the number of elements if user wants to use separate spec values
+for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data
+index.
+ *
+ * @author mwarner
+ */
+
+public class ThalesAWSSnowCADPFPEBulkUDFTester implements RequestStreamHandler {
+
+ private static byte[][] data;
+ private static AlgorithmParameterSpec[] spec;
+ private static Integer[] snowrownbrs;
+ private static final String BADDATATAG = new String ("9999999999999999");
+ private static int BATCHLIMIT = 10000;
+
+ // private static final Logger logger =
+ // Logger.getLogger(LambdaRequestStreamHandlerNbrEncyrptBulkFPE.class.getName());
+ private static final Gson gson = new Gson();
+
+ public static void main(String[] args) throws Exception {
+ ThalesAWSSnowCADPFPEBulkUDFTester nw2 = new ThalesAWSSnowCADPFPEBulkUDFTester();
+ String request = "{\r\n" + " \"resource\": \"/snowflake_proxy\",\r\n" + " \"path\": \"/snowflake_proxy\",\r\n"
+ + " \"httpMethod\": \"POST\",\r\n" + " \"headers\": {\r\n" + " \"Accept\": \"*/*\",\r\n"
+ + " \"Accept-Encoding\": \"gzip\",\r\n" + " \"Content-Encoding\": \"gzip\",\r\n"
+ + " \"Content-Type\": \"application/json\",\r\n"
+ + " \"Host\": \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\",\r\n"
+ + " \"sf-external-function-current-query-id\": \"01b02bdf-0001-7b58-0003-27d60056b5a6\",\r\n"
+ + " \"sf-external-function-format\": \"json\",\r\n"
+ + " \"sf-external-function-format-version\": \"1.0\",\r\n"
+ + " \"sf-external-function-name\": \"thales-aws-lambda-snow-cadp-encrypt-nbr\",\r\n"
+ + " \"sf-external-function-name-base64\": \"VEhBTEVTX0NBRFBfQVdTX1RPS0VOSVpFQ0hBUg==\",\r\n"
+ + " \"sf-external-function-query-batch-id\": \"01b02bdf-0001-7b58-0003-27d60056b5a6:1:1:0:0\",\r\n"
+ + " \"sf-external-function-return-type\": \"VARIANT\",\r\n"
+ + " \"sf-external-function-return-type-base64\": \"VkFSSUFOVA==\",\r\n"
+ + " \"sf-external-function-signature\": \"(B VARCHAR)\",\r\n"
+ + " \"sf-external-function-signature-base64\": \"KEIgVkFSQ0hBUik=\",\r\n"
+ + " \"User-Agent\": \"snowflake/1.0\",\r\n"
+ + " \"x-amz-content-sha256\": \"550cead700c849cd6f159aca9dd81083306bb5c919939c1e2c4f3f909f32332b\",\r\n"
+ + " \"x-amz-date\": \"20231107T142353Z\",\r\n"
+ + " \"x-amz-security-token\": \"IQoJb3JpZ2luX2VjEG8aCXVzLWVhc3QtMSJGMEQCICKJkvz9H9bTmSg3rRMNQYHnGeNzDyseSK3GxkGkxH6NAiBm/1kqxf5r25Z7154EYa93MaMOFIdMKS/b7MheulrJxCqHAwio//////////8BEAAaDDQ5NjQ4NDM0Mjc2NCIMCJwVrXJkl8t82Se4KtsC1mBjS4CLPcQEHJ+qRnw5HHbMTWx1DkJ6SeAjMWj/QypGy3yipoi20gY49xkbo4A0kzv7bty06Oa/VPp4id/hWdVEPVypuUQpyHKYdJAn/ZrnhvyqOMlKpGSg3157v1zPduOQOz9XvjpWC12NCaF3HAcj2/aH0DfCRacsAS+XwIWxNFOtrGbsbibrSJqFe2NqbvIbLU39cVLCD51cTMRMTXhdcRpU9JhXc8+I8+eWwsgpl823OrjwTwqDCnYs/9AY+yRCUDVjuLbKKQ+lXNWlaOMD6YfBzgNnjZZDbctx1lSMHG7HRHQrFIuZe0oKnVkbeF05CSxLBorfigKX8AF/Kyq5OhSnJPdTZKG6PbtWAub+JHaelZdEo3wDTMN04MkL+Nef+gV+g2vxejZiOc+v0HL0qXB1z2OLDqXfMm1zoACue3sYLqeK2jRjWqpEoPes7rU7jHOgkrWDgfkw7ZCpqgY6ngGFyKE++chqS6vGrZJ2e/HQL12UVqyxfT2NDRQZDA0gZmWnK5drpNJECXH9DeFcVqUnnDU77Ui5v6tXfUrq9cSOOYG68b2qCYiQ+TNkQPiMZNJmx4FYDL46RWtcxyvAMqaQz/5bhwf9W/Iw2QzYuSWj4FTC29H0eD1B5Frnq69Eg3A1A1bJ8m0LfDhT7gfK/rp7I2N0LSnYbhi2ksZU6A==\",\r\n"
+ + " \"X-Amzn-Trace-Id\": \"Root=1-654a4879-4aa55d5e4a5648a0268543d1\",\r\n"
+ + " \"X-Forwarded-For\": \"3.33.33.35\",\r\n" + " \"X-Forwarded-Port\": \"443\",\r\n"
+ + " \"X-Forwarded-Proto\": \"https\"\r\n" + " },\r\n" + " \"multiValueHeaders\": {\r\n"
+ + " \"Accept\": [\r\n" + " \"*/*\"\r\n" + " ],\r\n" + " \"Accept-Encoding\": [\r\n"
+ + " \"gzip\"\r\n" + " ],\r\n" + " \"Content-Encoding\": [\r\n" + " \"gzip\"\r\n"
+ + " ],\r\n" + " \"Content-Type\": [\r\n" + " \"application/json\"\r\n" + " ],\r\n"
+ + " \"Host\": [\r\n" + " \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-current-query-id\": [\r\n" + " \"sdfsdfsdff-27d60056b5a6\"\r\n"
+ + " ],\r\n" + " \"sf-external-function-format\": [\r\n" + " \"json\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-format-version\": [\r\n" + " \"1.0\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-name\": [\r\n" + " \"THALES_CADP_AWS_R\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-name-base64\": [\r\n" + " \"wewewewewewe==\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-query-batch-id\": [\r\n"
+ + " \"01b02bdf-0001-7b58-0003-27d60056b5a6:1:1:0:0\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-return-type\": [\r\n" + " \"VARIANT\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-return-type-base64\": [\r\n" + " \"VkFSSUFOVA==\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-signature\": [\r\n" + " \"(B VARCHAR)\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-signature-base64\": [\r\n" + " \"KEIgVkFSQ0hBUik=\"\r\n"
+ + " ],\r\n" + " \"User-Agent\": [\r\n" + " \"snowflake/1.0\"\r\n" + " ],\r\n"
+ + " \"x-amz-content-sha256\": [\r\n" + " \"550cead70rtrtrtrtb5c919939c1e2c4f3f909f32332b\"\r\n"
+ + " ],\r\n" + " \"x-amz-date\": [\r\n" + " \"20231107T142353Z\"\r\n" + " ],\r\n"
+ + " \"x-amz-security-token\": [\r\n"
+ + " \"IQoJb3JpZ2luX2VjEG8aCXVzLWVhc3QtMSJGMEQCICKJkvz9H9bTmSg3rRMNQYHnGeNzDyseSK3GxkGkxH6NAiBm/1kqxf5r25Z7154EYa93MaMOFIdMKS/b7MheulrJxCqHAwio//////////8BEAAaDDQ5NjQ4NDM0Mjc2NCIMCJwVrXJkl8t82Se4KtsC1mBjS4CLPcQEHJ+qRnw5HHbMTWx1DkJ6SeAjMWj/QypGy3yipoi20gY49xkbo4A0kzv7bty06Oa/VPp4id/hWdVEPVypuUQpyHKYdJAn/ZrnhvyqOMlKpGSg3157v1zPduOQOz9XvjpWC12NCaF3HAcj2/aH0DfCRacsAS+XwIWxNFOtrGbsbibrSJqFe2NqbvIbLU39cVLCD51cTMRMTXhdcRpU9JhXc8+I8+eWwsgpl823OrjwTwqDCnYs/9AY+yRCUDVjuLbKKQ+lXNWlaOMD6YfBzgNnjZZDbctx1lSMHG7HRHQrFIuZe0oKnVkbeF05CSxLBorfigKX8AF/Kyq5OhSnJPdTZKG6PbtWAub+JHaelZdEo3wDTMN04MkL+Nef+gV+g2vxejZiOc+v0HL0qXB1z2OLDqXfMm1zoACue3sYLqeK2jRjWqpEoPes7rU7jHOgkrWDgfkw7ZCpqgY6ngGFyKE++chqS6vGrZJ2e/HQL12UVqyxfT2NDRQZDA0gZmWnK5drpNJECXH9DeFcVqUnnDU77Ui5v6tXfUrq9cSOOYG68b2qCYiQ+TNkQPiMZNJmx4FYDL46RWtcxyvAMqaQz/5bhwf9W/Iw2QzYuSWj4FTC29H0eD1B5Frnq69Eg3A1A1bJ8m0LfDhT7gfK/rp7I2N0LSnYbhi2ksZU6A==\"\r\n"
+ + " ],\r\n" + " \"X-Amzn-Trace-Id\": [\r\n"
+ + " \"Root=1-654a4879-4aa55d5e4a5648a0268543d1\"\r\n" + " ],\r\n"
+ + " \"X-Forwarded-For\": [\r\n" + " \"3.34.44.35\"\r\n" + " ],\r\n"
+ + " \"X-Forwarded-Port\": [\r\n" + " \"443\"\r\n" + " ],\r\n"
+ + " \"X-Forwarded-Proto\": [\r\n" + " \"https\"\r\n" + " ]\r\n" + " },\r\n"
+ + " \"queryStringParameters\": null,\r\n" + " \"multiValueQueryStringParameters\": null,\r\n"
+ + " \"pathParameters\": null,\r\n" + " \"stageVariables\": null,\r\n" + " \"requestContext\": {\r\n"
+ + " \"resourceId\": \"erw8lc\",\r\n" + " \"resourcePath\": \"/snowflake_proxy\",\r\n"
+ + " \"httpMethod\": \"POST\",\r\n" + " \"extendedRequestId\": \"OCBC9FLtiYcEThQ=\",\r\n"
+ + " \"requestTime\": \"07/Nov/2023:14:23:53 +0000\",\r\n"
+ + " \"path\": \"/test/snowflake_proxy\",\r\n" + " \"accountId\": \"455555555564\",\r\n"
+ + " \"protocol\": \"HTTP/1.1\",\r\n" + " \"stage\": \"test\",\r\n"
+ + " \"domainPrefix\": \"asdfsdfsfff\",\r\n" + " \"requestTimeEpoch\": 1699367033023,\r\n"
+ + " \"requestId\": \"f53460af-d302-4e3a-b837-f5a9785badcb\",\r\n" + " \"identity\": {\r\n"
+ + " \"cognitoIdentityPoolId\": null,\r\n" + " \"accountId\": \"34343434\",\r\n"
+ + " \"cognitoIdentityId\": null,\r\n" + " \"caller\": \"ARODFDFDFFDFDFX7G:snowflake\",\r\n"
+ + " \"sourceIp\": \"3.33.44.35\",\r\n" + " \"principalOrgId\": null,\r\n"
+ + " \"accessKey\": \"FAKE\",\r\n" + " \"cognitoAuthenticationType\": null,\r\n"
+ + " \"cognitoAuthenticationProvider\": null,\r\n"
+ + " \"userArn\": \"arn:aws:sts::496484342764:assumed-role/snowflakerole/snowflake\",\r\n"
+ + " \"userAgent\": \"snowflake/1.0\",\r\n"
+ + " \"user\": \"shawnscanlan@snowflakecomputing.com\"\r\n" + " },\r\n"
+ + " \"domainName\": \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\",\r\n"
+ + " \"apiId\": \"asdfsdfsfff\"\r\n" + " },\r\n"
+ + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"0\\\"]\\n,[1,\\\"null\\\"]\\n,[2,\\\"45\\\"]\\n,[3,\\\"32323232323232\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n"
+ + " \"isBase64Encoded\": false\r\n" + "}";
+//+ " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4545554545675\\\"]\\n,[1,\\\"124313145756\\\"]\\n,[2,\\\"584785473839\\\"]\\n,[3,\\\"32323232323232\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n"
+ // " \"user\": \"ADFDFDFFG:snowflake\"\r\n" +
+// + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4\\\"]\\n,[1,\\\"12431-3145-756\\\"]\\n,[2,\\\"null\\\"]\\n,[3,\\\"\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n"
+ // nw2.handleRequest(request, null, null);
+ //backup
+ // + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"454-55545-45675\\\"]\\n,[1,\\\"1243-131457-56\\\"]\\n,[2,\\\"584785473839\\\"]\\n,[3,\\\"32323232323232\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n"
+
+ nw2.handleRequest2(request, null, null);
+ }
+
+
+ public void handleRequest2(String inputStream, OutputStream outputStream, Context context) throws IOException {
+ // context.getLogger().log("Input: " + inputStream);
+ // String input = IOUtils.toString(inputStream, "UTF-8");
+ String input = inputStream;
+ Map encryptedErrorMapTotal = new HashMap();
+
+ String tweakAlgo = null;
+ String tweakData = null;
+ String snowflakereturnstring = null;
+ JsonObject body = null;
+ int statusCode = 200;
+ int numberofchunks = 0;
+
+ String callerStr = null;
+
+ // https://www.baeldung.com/java-aws-lambda
+
+ JsonObject snowflakeinput = null;
+
+ JsonArray snowflakedata = null;
+
+ NAESession session = null;
+ String keyName = "testfaas";
+ // String keyName = System.getenv("CMKEYNAME");
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be
+ // returned when the user above does not have access to the key and if doing a lookup in the userset
+ // and the user does not exist. If returnciphertextforuserwithnokeyaccess = no then an error will be
+ // returned to the query, else the results set will provide ciphertext.
+ // yes/no
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB?
+ // yes/no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it
+ // is the userset in CM but could be a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ //datatype - data type in db/actual data format/return type. valid values are char, charint, nbr, charintchar
+ String datatype = System.getenv("datatype");
+ //mode of operation valid values are : encrypt or decrypt
+ String mode = System.getenv("mode");
+
+ int cipherType = 0;
+ if (mode.equals("encrypt"))
+ cipherType = Cipher.ENCRYPT_MODE;
+ else
+ cipherType = Cipher.DECRYPT_MODE;
+
+
+ try {
+
+ // int batchsize = Integer.parseInt(System.getenv("BATCHSIZE"));
+
+
+ JsonElement rootNode = JsonParser.parseString(input).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ snowflakeinput = rootNode.getAsJsonObject();
+ if (snowflakeinput.isJsonObject()) {
+ // For some reason when using snowflake it adds \n and \ to quotes in json.
+ // the JsonParser.parseString(input).getAsJsonObject(); is supposed to remove
+ // all of those
+ // characters but it does not do it for snowflake json.
+ JsonElement bodyele = snowflakeinput.get("body");
+ String bodystr = bodyele.getAsString().replaceAll(System.lineSeparator(), "");
+ bodystr = bodystr.replaceAll("\\\\", "");
+ // System.out.println("bodystr after replace" + bodystr);
+ body = gson.fromJson(bodystr, JsonObject.class);
+ snowflakedata = body.getAsJsonArray("data");
+
+ JsonObject requestContext = snowflakeinput.getAsJsonObject("requestContext");
+
+ if (requestContext != null) {
+ JsonObject identity = requestContext.getAsJsonObject("identity");
+
+ if (identity != null) {
+ callerStr = identity.get("user").getAsString();
+ System.out.println("user: " + callerStr);
+ } else {
+ System.out.println("Identity not found.");
+ }
+ } else {
+ System.out.println("Request context not found.");
+ }
+
+ if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group
+
+ boolean founduserinuserset = findUserInUserSet(callerStr, userName, password, usersetID,
+ userSetLookupIP);
+ // System.out.println("Found User " + founduserinuserset);
+ if (!founduserinuserset)
+ throw new CustomException("1001, User Not in User Set", 1001);
+
+ } else {
+ usersetlookupbool = false;
+ }
+
+ } else {
+ System.out.println("eerror");
+
+ }
+ }
+
+ System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
+ "D:\\product\\Build\\CADP_for_JAVA.properties");
+
+ /*
+ * System.setProperty(
+ * "com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
+ * "CADP_for_JAVA.properties"); IngrianProvider builder = new
+ * Builder().addConfigFileInputStream(
+ * getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).
+ * build();
+ */
+
+ session = NAESession.getSession(userName, password.toCharArray());
+ NAEKey key = NAEKey.getSecretKey(keyName, session);
+ String algorithm = null;
+
+ if (datatype.equals("char"))
+ algorithm = "FPE/FF1/CARD62";
+ else if (datatype.equals("charint"))
+ algorithm = "FPE/FF1/CARD10";
+ else
+ algorithm = "FPE/FF1/CARD10";
+
+
+ int row_number = 0;
+
+ AbstractNAECipher thalesCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider");
+ int batchsize = 5;
+ System.out.println("Batchsize = " + batchsize);
+ int numberOfLines = snowflakedata.size();
+ int totalRowsLeft = numberOfLines;
+
+ if (batchsize > numberOfLines)
+ batchsize = numberOfLines;
+ if (batchsize >= BATCHLIMIT)
+ batchsize = BATCHLIMIT;
+
+ FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo).build();
+
+ spec = new FPEParameterAndFormatSpec[batchsize];
+ data = new byte[batchsize][];
+ snowrownbrs = new Integer[batchsize];
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+ JsonArray innerDataArray = new JsonArray();
+
+ thalesCipher.init(cipherType, key, spec[0]);
+
+ int i = 0;
+ int count = 0;
+ boolean newchunk = true;
+ int dataIndex = 0;
+ int specIndex = 0;
+ int snowRowIndex = 0;
+ String sensitive = null;
+ while (i < numberOfLines) {
+ int index = 0;
+
+ if (newchunk) {
+
+ if (totalRowsLeft < batchsize) {
+ spec = new FPEParameterAndFormatSpec[totalRowsLeft];
+ data = new byte[totalRowsLeft][];
+ snowrownbrs = new Integer[totalRowsLeft];
+ } else {
+ spec = new FPEParameterAndFormatSpec[batchsize];
+ data = new byte[batchsize][];
+ snowrownbrs = new Integer[batchsize];
+ }
+ newchunk = false;
+ }
+
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ for (int j = 0; j < snowflakerow.size(); j++) {
+
+ if (j == 1) {
+
+ // FPE example
+ sensitive = checkValid(snowflakerow);
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ sensitive = BADDATATAG+sensitive;
+ data[dataIndex++] = sensitive.getBytes();
+ // Can not return number since a leading 0 will not work.
+ // innerDataArray.add(new BigInteger(sensitive));
+ } else
+ data[dataIndex++] = BADDATATAG.getBytes();
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+ data[dataIndex++] = sensitive.getBytes();
+ } else {
+ data[dataIndex++] = sensitive.getBytes();
+ }
+ spec[specIndex++] = param;
+
+ } else {
+ data[dataIndex++] = sensitive.getBytes();
+ spec[specIndex++] = param;
+ }
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ row_number = snowflakecolumn.getAsInt();
+ snowrownbrs[snowRowIndex++] = row_number;
+
+ }
+
+ }
+
+ if (count == batchsize - 1) {
+
+ // Map to store exceptions while encryption
+ Map encryptedErrorMap = new HashMap();
+
+ byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap);
+
+ for (Map.Entry entry : encryptedErrorMap.entrySet()) {
+ Integer mkey = entry.getKey();
+ String mvalue = entry.getValue();
+ encryptedErrorMapTotal.put(mkey, mvalue);
+ }
+
+ for (int enc = 0; enc < encryptedData.length; enc++) {
+
+ innerDataArray.add(snowrownbrs[enc]);
+ innerDataArray.add(new String(encryptedData[enc]));
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ index++;
+
+ }
+
+ numberofchunks++;
+ newchunk = true;
+ count = 0;
+ dataIndex = 0;
+ specIndex = 0;
+ snowRowIndex = 0;
+ } else
+ count++;
+
+ totalRowsLeft--;
+ i++;
+ }
+
+ if (count > 0) {
+ numberofchunks++;
+ int index = 0;
+ Map encryptedErrorMap = new HashMap();
+ byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap);
+ for (int enc = 0; enc < encryptedData.length; enc++) {
+ innerDataArray.add(snowrownbrs[enc]);
+ innerDataArray.add(new String(encryptedData[enc]));
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ index++;
+
+ }
+ }
+
+
+ bodyObject.add("data", dataArray);
+ JsonObject inputJsonObject = new JsonObject();
+ String bodyString = bodyObject.toString();
+ inputJsonObject.addProperty("statusCode", 200);
+ inputJsonObject.addProperty("body", bodyString);
+
+ snowflakereturnstring = inputJsonObject.toString();
+ //System.out.println("snowflakereturnstring = " + snowflakereturnstring);
+
+
+ } catch (
+
+ Exception e) {
+ System.out.println("in exception with " + e.getMessage());
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401") || (e.getMessage().contains("1001") || (e.getMessage().contains("1002"))) ) {
+
+ try {
+ snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, true, null,datatype);
+ } catch (IllegalBlockSizeException e1) {
+ // TODO Auto-generated catch block
+ e1.printStackTrace();
+ } catch (BadPaddingException e1) {
+ // TODO Auto-generated catch block
+ e1.printStackTrace();
+ }
+
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } finally {
+ if (session != null) {
+ session.closeSession();
+ }
+ }
+ System.out.println("number of chunks = " + numberofchunks);
+ System.out.println("snowflakereturnstring = " + snowflakereturnstring);
+
+
+ }
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+
+ boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
+
+ return founduserinuserset;
+
+ }
+
+ public String checkValid(JsonArray snowrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (snowrow != null && snowrow.size() > 0) {
+ JsonElement element = snowrow.get(1);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer snowflakereturndatasb = new StringBuffer();
+
+ snowflakereturndatasb.append("{ \"statusCode\":");
+ snowflakereturndatasb.append(statusCode);
+ snowflakereturndatasb.append(",");
+ snowflakereturndatasb.append(" \"body\": {");
+ snowflakereturndatasb.append(" \"data\": [");
+ snowflakereturndatasb.append("] }}");
+ System.out.println("in exception with ");
+ return snowflakereturndatasb.toString();
+ }
+
+ public String formatReturnValue(int statusCode, JsonArray snowflakedata, boolean error, Cipher thalesCipher,
+ String datatype) throws IllegalBlockSizeException, BadPaddingException {
+ int row_number = 0;
+
+ String encdata = null;
+ String sensitive = null;
+
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+ JsonArray innerDataArray = new JsonArray();
+ int nbrofrows = snowflakedata.size();
+ for (int i = 0; i < nbrofrows; i++) {
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+ for (int j = 0; j < snowflakerow.size(); j++) {
+ if (j == 1) {
+ sensitive = checkValid(snowflakerow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ innerDataArray.add(sensitive);
+ // Can not return number since a leading 0 will not work.
+ // innerDataArray.add(new BigInteger(sensitive));
+ } else
+ innerDataArray.add(BADDATATAG);
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+ innerDataArray.add("");
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ innerDataArray.add(sensitive);
+ } else {
+ innerDataArray.add(sensitive);
+ }
+
+ } else {
+ if (!error) {
+ byte[] outbuf = thalesCipher.doFinal(sensitive.getBytes());
+ encdata = new String(outbuf);
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ innerDataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ innerDataArray.add(encdata);
+ }
+ } else {
+ encdata = sensitive;
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ innerDataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ innerDataArray.add(encdata);
+ }
+ }
+ }
+
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ row_number = snowflakecolumn.getAsInt();
+ innerDataArray.add(row_number);
+ }
+ }
+ }
+
+ bodyObject.add("data", dataArray);
+ JsonObject inputJsonObject = new JsonObject();
+ String bodyString = bodyObject.toString();
+ inputJsonObject.addProperty("statusCode", 200);
+ inputJsonObject.addProperty("body", bodyString);
+
+ String formattedStringnew = inputJsonObject.toString();
+
+ return formattedStringnew;
+
+ }
+
+ @Override
+ public void handleRequest(InputStream input, OutputStream output, Context context) throws IOException {
+ // TODO Auto-generated method stub
+
+ }
+
+
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCADPFPEUDFTester.java b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCADPFPEUDFTester.java
new file mode 100644
index 00000000..48e8b7ae
--- /dev/null
+++ b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCADPFPEUDFTester.java
@@ -0,0 +1,433 @@
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.math.BigInteger;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec;
+import com.ingrian.security.nae.IngrianProvider;
+import com.ingrian.security.nae.NAEKey;
+import com.ingrian.security.nae.NAESession;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
+import com.ingrian.security.nae.IngrianProvider.Builder;
+
+/* This sample AWS Lambda Function is used to implement a Snowflake Database User Defined Function(UDF). /*
+ * It is an example of how to use Thales Cipher Trust Application Data Protection (CADP)
+ * to protect sensitive data in a column. This example uses
+ * Format Preserve Encryption (FPE) to maintain the original format of the data
+ * so applications or business intelligence tools do not have to change in order
+ * to use these columns.
+ *
+* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
+* for all possible data sizes and combinations of encryption algorithms and IV, etc.
+* Was tested with CM 2.14 & CADP CADP 8.15.0.001
+* For more information on CADP see link below.
+https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html
+* For more information on Snowflake External Functions see link below.
+https://docs.snowflake.com/en/sql-reference/external-functions-creating-aws
+ *
+ */
+
+public class ThalesAWSSnowCADPFPEUDFTester implements RequestStreamHandler {
+// private static final Logger logger = Logger.getLogger(LambdaRequestStreamHandlerCharEncryptFPE.class.getName());
+ private static final Gson gson = new Gson();
+ private static final String BADDATATAG = new String ("9999999999999999");
+ public static void main(String[] args) throws Exception
+
+ {
+ ThalesAWSSnowCADPFPEUDFTester nw2 = new ThalesAWSSnowCADPFPEUDFTester();
+ String request = "{\r\n" + " \"resource\": \"/snowflake_proxy\",\r\n" + " \"path\": \"/snowflake_proxy\",\r\n"
+ + " \"httpMethod\": \"POST\",\r\n" + " \"headers\": {\r\n" + " \"Accept\": \"*/*\",\r\n"
+ + " \"Accept-Encoding\": \"gzip\",\r\n" + " \"Content-Encoding\": \"gzip\",\r\n"
+ + " \"Content-Type\": \"application/json\",\r\n"
+ + " \"Host\": \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\",\r\n"
+ + " \"sf-external-function-current-query-id\": \"01b02bdf-0001-7b58-0003-27d60056b5a6\",\r\n"
+ + " \"sf-external-function-format\": \"json\",\r\n"
+ + " \"sf-external-function-format-version\": \"1.0\",\r\n"
+ + " \"sf-external-function-name\": \"thales-aws-lambda-snow-cadp-encrypt-nbr\",\r\n"
+ + " \"sf-external-function-name-base64\": \"VEhBTEVTX0NBRFBfQVdTX1RPS0VOSVpFQ0hBUg==\",\r\n"
+ + " \"sf-external-function-query-batch-id\": \"01b02bdf-0001-7b58-0003-27d60056b5a6:1:1:0:0\",\r\n"
+ + " \"sf-external-function-return-type\": \"VARIANT\",\r\n"
+ + " \"sf-external-function-return-type-base64\": \"VkFSSUFOVA==\",\r\n"
+ + " \"sf-external-function-signature\": \"(B VARCHAR)\",\r\n"
+ + " \"sf-external-function-signature-base64\": \"KEIgVkFSQ0hBUik=\",\r\n"
+ + " \"User-Agent\": \"snowflake/1.0\",\r\n"
+ + " \"x-amz-content-sha256\": \"550cead700c849cd6f159aca9dd81083306bb5c919939c1e2c4f3f909f32332b\",\r\n"
+ + " \"x-amz-date\": \"20231107T142353Z\",\r\n"
+ + " \"x-amz-security-token\": \"IQoJb3JpZ2luX2VjEG8aCXVzLWVhc3QtMSJGMEQCICKJkvz9H9bTmSg3rRMNQYHnGeNzDyseSK3GxkGkxH6NAiBm/1kqxf5r25Z7154EYa93MaMOFIdMKS/b7MheulrJxCqHAwio//////////8BEAAaDDQ5NjQ4NDM0Mjc2NCIMCJwVrXJkl8t82Se4KtsC1mBjS4CLPcQEHJ+qRnw5HHbMTWx1DkJ6SeAjMWj/QypGy3yipoi20gY49xkbo4A0kzv7bty06Oa/VPp4id/hWdVEPVypuUQpyHKYdJAn/ZrnhvyqOMlKpGSg3157v1zPduOQOz9XvjpWC12NCaF3HAcj2/aH0DfCRacsAS+XwIWxNFOtrGbsbibrSJqFe2NqbvIbLU39cVLCD51cTMRMTXhdcRpU9JhXc8+I8+eWwsgpl823OrjwTwqDCnYs/9AY+yRCUDVjuLbKKQ+lXNWlaOMD6YfBzgNnjZZDbctx1lSMHG7HRHQrFIuZe0oKnVkbeF05CSxLBorfigKX8AF/Kyq5OhSnJPdTZKG6PbtWAub+JHaelZdEo3wDTMN04MkL+Nef+gV+g2vxejZiOc+v0HL0qXB1z2OLDqXfMm1zoACue3sYLqeK2jRjWqpEoPes7rU7jHOgkrWDgfkw7ZCpqgY6ngGFyKE++chqS6vGrZJ2e/HQL12UVqyxfT2NDRQZDA0gZmWnK5drpNJECXH9DeFcVqUnnDU77Ui5v6tXfUrq9cSOOYG68b2qCYiQ+TNkQPiMZNJmx4FYDL46RWtcxyvAMqaQz/5bhwf9W/Iw2QzYuSWj4FTC29H0eD1B5Frnq69Eg3A1A1bJ8m0LfDhT7gfK/rp7I2N0LSnYbhi2ksZU6A==\",\r\n"
+ + " \"X-Amzn-Trace-Id\": \"Root=1-654a4879-4aa55d5e4a5648a0268543d1\",\r\n"
+ + " \"X-Forwarded-For\": \"3.33.33.35\",\r\n" + " \"X-Forwarded-Port\": \"443\",\r\n"
+ + " \"X-Forwarded-Proto\": \"https\"\r\n" + " },\r\n" + " \"multiValueHeaders\": {\r\n"
+ + " \"Accept\": [\r\n" + " \"*/*\"\r\n" + " ],\r\n" + " \"Accept-Encoding\": [\r\n"
+ + " \"gzip\"\r\n" + " ],\r\n" + " \"Content-Encoding\": [\r\n" + " \"gzip\"\r\n"
+ + " ],\r\n" + " \"Content-Type\": [\r\n" + " \"application/json\"\r\n" + " ],\r\n"
+ + " \"Host\": [\r\n" + " \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-current-query-id\": [\r\n" + " \"sdfsdfsdff-27d60056b5a6\"\r\n"
+ + " ],\r\n" + " \"sf-external-function-format\": [\r\n" + " \"json\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-format-version\": [\r\n" + " \"1.0\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-name\": [\r\n" + " \"THALES_CADP_AWS_R\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-name-base64\": [\r\n" + " \"wewewewewewe==\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-query-batch-id\": [\r\n"
+ + " \"01b02bdf-0001-7b58-0003-27d60056b5a6:1:1:0:0\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-return-type\": [\r\n" + " \"VARIANT\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-return-type-base64\": [\r\n" + " \"VkFSSUFOVA==\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-signature\": [\r\n" + " \"(B VARCHAR)\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-signature-base64\": [\r\n" + " \"KEIgVkFSQ0hBUik=\"\r\n"
+ + " ],\r\n" + " \"User-Agent\": [\r\n" + " \"snowflake/1.0\"\r\n" + " ],\r\n"
+ + " \"x-amz-content-sha256\": [\r\n" + " \"550cead70rtrtrtrtb5c919939c1e2c4f3f909f32332b\"\r\n"
+ + " ],\r\n" + " \"x-amz-date\": [\r\n" + " \"20231107T142353Z\"\r\n" + " ],\r\n"
+ + " \"x-amz-security-token\": [\r\n"
+ + " \"IQoJb3JpZ2luX2VjEG8aCXVzLWVhc3QtMSJGMEQCICKJkvz9H9bTmSg3rRMNQYHnGeNzDyseSK3GxkGkxH6NAiBm/1kqxf5r25Z7154EYa93MaMOFIdMKS/b7MheulrJxCqHAwio//////////8BEAAaDDQ5NjQ4NDM0Mjc2NCIMCJwVrXJkl8t82Se4KtsC1mBjS4CLPcQEHJ+qRnw5HHbMTWx1DkJ6SeAjMWj/QypGy3yipoi20gY49xkbo4A0kzv7bty06Oa/VPp4id/hWdVEPVypuUQpyHKYdJAn/ZrnhvyqOMlKpGSg3157v1zPduOQOz9XvjpWC12NCaF3HAcj2/aH0DfCRacsAS+XwIWxNFOtrGbsbibrSJqFe2NqbvIbLU39cVLCD51cTMRMTXhdcRpU9JhXc8+I8+eWwsgpl823OrjwTwqDCnYs/9AY+yRCUDVjuLbKKQ+lXNWlaOMD6YfBzgNnjZZDbctx1lSMHG7HRHQrFIuZe0oKnVkbeF05CSxLBorfigKX8AF/Kyq5OhSnJPdTZKG6PbtWAub+JHaelZdEo3wDTMN04MkL+Nef+gV+g2vxejZiOc+v0HL0qXB1z2OLDqXfMm1zoACue3sYLqeK2jRjWqpEoPes7rU7jHOgkrWDgfkw7ZCpqgY6ngGFyKE++chqS6vGrZJ2e/HQL12UVqyxfT2NDRQZDA0gZmWnK5drpNJECXH9DeFcVqUnnDU77Ui5v6tXfUrq9cSOOYG68b2qCYiQ+TNkQPiMZNJmx4FYDL46RWtcxyvAMqaQz/5bhwf9W/Iw2QzYuSWj4FTC29H0eD1B5Frnq69Eg3A1A1bJ8m0LfDhT7gfK/rp7I2N0LSnYbhi2ksZU6A==\"\r\n"
+ + " ],\r\n" + " \"X-Amzn-Trace-Id\": [\r\n"
+ + " \"Root=1-654a4879-4aa55d5e4a5648a0268543d1\"\r\n" + " ],\r\n"
+ + " \"X-Forwarded-For\": [\r\n" + " \"3.34.44.35\"\r\n" + " ],\r\n"
+ + " \"X-Forwarded-Port\": [\r\n" + " \"443\"\r\n" + " ],\r\n"
+ + " \"X-Forwarded-Proto\": [\r\n" + " \"https\"\r\n" + " ]\r\n" + " },\r\n"
+ + " \"queryStringParameters\": null,\r\n" + " \"multiValueQueryStringParameters\": null,\r\n"
+ + " \"pathParameters\": null,\r\n" + " \"stageVariables\": null,\r\n" + " \"requestContext\": {\r\n"
+ + " \"resourceId\": \"erw8lc\",\r\n" + " \"resourcePath\": \"/snowflake_proxy\",\r\n"
+ + " \"httpMethod\": \"POST\",\r\n" + " \"extendedRequestId\": \"OCBC9FLtiYcEThQ=\",\r\n"
+ + " \"requestTime\": \"07/Nov/2023:14:23:53 +0000\",\r\n"
+ + " \"path\": \"/test/snowflake_proxy\",\r\n" + " \"accountId\": \"455555555564\",\r\n"
+ + " \"protocol\": \"HTTP/1.1\",\r\n" + " \"stage\": \"test\",\r\n"
+ + " \"domainPrefix\": \"asdfsdfsfff\",\r\n" + " \"requestTimeEpoch\": 1699367033023,\r\n"
+ + " \"requestId\": \"f53460af-d302-4e3a-b837-f5a9785badcb\",\r\n" + " \"identity\": {\r\n"
+ + " \"cognitoIdentityPoolId\": null,\r\n" + " \"accountId\": \"34343434\",\r\n"
+ + " \"cognitoIdentityId\": null,\r\n" + " \"caller\": \"ARODFDFDFFDFDFX7G:snowflake\",\r\n"
+ + " \"sourceIp\": \"3.33.44.35\",\r\n" + " \"principalOrgId\": null,\r\n"
+ + " \"accessKey\": \"FAKE\",\r\n" + " \"cognitoAuthenticationType\": null,\r\n"
+ + " \"cognitoAuthenticationProvider\": null,\r\n"
+ + " \"userArn\": \"arn:aws:sts::496484342764:assumed-role/snowflakerole/snowflake\",\r\n"
+ + " \"userAgent\": \"snowflake/1.0\",\r\n"
+ + " \"user\": \"shawnscanlan@snowflakecomputing.com\"\r\n" + " },\r\n"
+ + " \"domainName\": \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\",\r\n"
+ + " \"apiId\": \"asdfsdfsfff\"\r\n" + " },\r\n"
+ + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"WL8ZUOD1kM\\\"]\\n]\\n}\",\r\n"
+ + " \"isBase64Encoded\": false\r\n" + "}";
+
+ //+ " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4545554545675\\\"]\\n,[1,\\\"124313145756\\\"]\\n,[2,\\\"584785473839\\\"]\\n,[3,\\\"32323232323232\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n"
+ //+ " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"markwarner\\\"]\\n]\\n}\",\r\n"
+ // decrypt 2508004056582 + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4545554545675\\\"]\\n]\\n}\",\r\n"
+ // + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4545554545675\\\"]\\n]\\n}\",\r\n"
+// + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4545554-5456-75\\\"]\\n]\\n}\",\r\n"
+ // "body":
+ // "{\n\"data\":[\n[0,\"ndorgan5@sf_tuts.com\"],\n[1,\"cdevereu6@sf_tuts.co.au\"],\n[2,\"gglaserman7@sf_tuts.com\"],\n[3,\"icasemore8@sf_tuts.com\"],\n[4,\"chovie9@sf_tuts.com\"],\n[5,\"afeatherstona@sf_tuts.com\"],\n[6,\"hanneslieb@sf_tuts.com\"],\n[7,\"bciccoc@sf_tuts.com\"],\n[8,\"bdurnalld@sf_tuts.com\"],\n[9,\"kmacconnale@sf_tuts.com\"],\n[10,\"wsizeyf@sf_tuts.com\"],\n[11,\"dmcgowrang@sf_tuts.com\"],\n[12,\"cbedderh@sf_tuts.co.au\"],\n[13,\"davoryi@sf_tuts.com\"],\n[14,\"rtalmadgej@sf_tuts.co.uk\"],\n[15,\"lboissier@sf_tuts.com\"],\n[16,\"ihanks1@sf_tuts.com\"],\n[17,\"alaudham2@sf_tuts.com\"],\n[18,\"ecornner3@sf_tuts.com\"],\n[19,\"hgoolding4@sf_tuts.com\"],\n[20,\"adavidovitsk@sf_tuts.com\"],\n[21,\"vshermorel@sf_tuts.com\"],\n[22,\"rmattysm@sf_tuts.com\"],\n[23,\"soluwatoyinn@sf_tuts.com\"],\n[24,\"gbassfordo@sf_tuts.co.uk\"]\n]\n}",
+ // "isBase64Encoded": false
+
+ // " \"user\": \"ADFDFDFFG:snowflake\"\r\n" +
+ // + " \"user\": \"shawnscanlan@snowflakecomputing.com\"\r\n" + " },\r\n"
+ nw2.handleRequest(request, null, null);
+
+ }
+
+ public void handleRequest(String inputStream, OutputStream outputStream, Context context)
+ throws IOException, IllegalBlockSizeException, BadPaddingException {
+
+ String input = inputStream;
+ String tweakAlgo = null;
+ String tweakData = null;
+ String snowflakereturnstring = null;
+ JsonObject body = null;
+ int statusCode = 200;
+
+ // https://www.baeldung.com/java-aws-lambda
+ JsonObject snowflakeinput = null;
+ String callerStr = null;
+ JsonArray snowflakedata = null;
+
+ NAESession session = null;
+
+ String keyName = "testfaas";
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be
+ // returned when the user above does not have access to the key and if doing a lookup in the userset
+ // and the user does not exist. If returnciphertextforuserwithnokeyaccess = no then an error will be
+ // returned to the query, else the results set will provide ciphertext.
+ // yes/no
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB?
+ // yes/no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it
+ // is the userset in CM but could be a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ //datatype - data type in db/actual data format/return type. valid values are char, charint, nbr, charintchar
+ String datatype = System.getenv("datatype");
+ //mode of operation valid values are : encrypt or decrypt
+ String mode = System.getenv("mode");
+
+ int cipherType = 0;
+ if (mode.equals("encrypt"))
+ cipherType = Cipher.ENCRYPT_MODE;
+ else
+ cipherType = Cipher.DECRYPT_MODE;
+
+
+ try {
+
+ JsonElement rootNode = JsonParser.parseString(input).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ snowflakeinput = rootNode.getAsJsonObject();
+ if (snowflakeinput.isJsonObject()) {
+ // For some reason when using snowflake it adds \n and \ to quotes in json.
+ // the JsonParser.parseString(input).getAsJsonObject(); is supposed to remove
+ // all of those
+ // characters but it does not do it for snowflake json.
+ JsonElement bodyele = snowflakeinput.get("body");
+ String bodystr = bodyele.getAsString().replaceAll(System.lineSeparator(), "");
+ // System.out.println("bodystr before replace" + bodystr );
+ bodystr = bodystr.replaceAll("\\\\", "");
+ // System.out.println("bodystr after replace" + bodystr );
+ body = gson.fromJson(bodystr, JsonObject.class);
+ snowflakedata = body.getAsJsonArray("data");
+ JsonObject requestContext = snowflakeinput.getAsJsonObject("requestContext");
+
+ if (requestContext != null) {
+ JsonObject identity = requestContext.getAsJsonObject("identity");
+
+ if (identity != null) {
+ callerStr = identity.get("user").getAsString();
+ System.out.println("user: " + callerStr);
+ } else {
+ System.out.println("Identity not found.");
+ }
+ } else {
+ System.out.println("Request context not found.");
+ }
+
+ if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group
+
+ boolean founduserinuserset = findUserInUserSet(callerStr, userName, password, usersetID,
+ userSetLookupIP);
+ // System.out.println("Found User " + founduserinuserset);
+ if (!founduserinuserset)
+ throw new CustomException("1001, User Not in User Set", 1001);
+
+ } else {
+ usersetlookupbool = false;
+ }
+
+ } else {
+ System.out.println("eerror");
+
+ }
+ }
+
+ System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
+ "D:\\product\\Build\\CADP_for_JAVA.properties");
+
+ // System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
+ // "CADP_for_JAVA.properties");
+ // IngrianProvider builder = new Builder().addConfigFileInputStream(
+ // getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
+ session = NAESession.getSession(userName, password.toCharArray());
+ NAEKey key = NAEKey.getSecretKey(keyName, session);
+
+ String algorithm = null;
+
+ if (datatype.equals("char"))
+ algorithm = "FPE/FF1/CARD62";
+ else if (datatype.equals("charint"))
+ algorithm = "FPE/FF1/CARD10";
+ else
+ algorithm = "FPE/FF1/CARD10";
+
+ FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
+ .build();
+ ///
+ // ivSpec = param;
+ Cipher thalesCipher = Cipher.getInstance(algorithm, "IngrianProvider");
+
+ thalesCipher.init(cipherType, key, param);
+ snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, false, thalesCipher, datatype);
+
+ } catch (Exception e) {
+
+ System.out.println("in exception with " + e.getMessage());
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
+
+ snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, true, null, datatype);
+
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+
+ } finally {
+ if (session != null) {
+ session.closeSession();
+ }
+ }
+ System.out.println("results" + snowflakereturnstring);
+ // outputStream.write(snowflakereturnstring.getBytes());
+
+ }
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer snowflakereturndatasb = new StringBuffer();
+
+ snowflakereturndatasb.append("{ \"statusCode\":");
+ snowflakereturndatasb.append(statusCode);
+ snowflakereturndatasb.append(",");
+ snowflakereturndatasb.append(" \"body\": {");
+ snowflakereturndatasb.append(" \"data\": [");
+ snowflakereturndatasb.append("] }}");
+ System.out.println("in exception with ");
+ return snowflakereturndatasb.toString();
+ }
+
+ public String checkValid(JsonArray snowrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (snowrow != null && snowrow.size() > 0) {
+ JsonElement element = snowrow.get(1);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ public String formatReturnValue(int statusCode, JsonArray snowflakedata, boolean error, Cipher thalesCipher,
+ String datatype) throws IllegalBlockSizeException, BadPaddingException {
+ int row_number = 0;
+
+ String encdata = null;
+ String sensitive = null;
+
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+ JsonArray innerDataArray = new JsonArray();
+ int nbrofrows = snowflakedata.size();
+ for (int i = 0; i < nbrofrows; i++) {
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+ for (int j = 0; j < snowflakerow.size(); j++) {
+ if (j == 1) {
+ sensitive = checkValid(snowflakerow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ innerDataArray.add(sensitive);
+ // Can not return number since a leading 0 will not work.
+ // innerDataArray.add(new BigInteger(sensitive));
+ } else
+ innerDataArray.add(BADDATATAG);
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+ innerDataArray.add("");
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ innerDataArray.add(sensitive);
+ } else {
+ innerDataArray.add(sensitive);
+ }
+
+ } else {
+ if (!error) {
+ byte[] outbuf = thalesCipher.doFinal(sensitive.getBytes());
+ encdata = new String(outbuf);
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ innerDataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ innerDataArray.add(encdata);
+ }
+ } else {
+ encdata = sensitive;
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ innerDataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ innerDataArray.add(encdata);
+ }
+ }
+ }
+
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ row_number = snowflakecolumn.getAsInt();
+ innerDataArray.add(row_number);
+ }
+ }
+ }
+
+ bodyObject.add("data", dataArray);
+ JsonObject inputJsonObject = new JsonObject();
+ String bodyString = bodyObject.toString();
+ inputJsonObject.addProperty("statusCode", 200);
+ inputJsonObject.addProperty("body", bodyString);
+
+ String formattedStringnew = inputJsonObject.toString();
+
+ return formattedStringnew;
+
+ }
+
+ @Override
+ public void handleRequest(InputStream input, OutputStream output, Context context) throws IOException {
+ // TODO Auto-generated method stub
+
+ }
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+
+ boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
+
+ return founduserinuserset;
+
+ }
+
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCRDPFPEBulkUDFTester.java b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCRDPFPEBulkUDFTester.java
new file mode 100644
index 00000000..19d16aa3
--- /dev/null
+++ b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCRDPFPEBulkUDFTester.java
@@ -0,0 +1,706 @@
+
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.math.BigInteger;
+import java.security.spec.AlgorithmParameterSpec;
+import java.util.HashMap;
+import java.util.Map;
+
+
+import org.apache.commons.io.IOUtils;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+
+import okhttp3.MediaType;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
+/* This sample AWS Lambda Function is used to implement a Snowflake Database User Defined Function(UDF). /*
+ * It is an example of how to use Thales Cipher Trust Application Data Protection (CADP)
+ * to protect sensitive data in a column. This example uses
+ * Format Preserve Encryption (FPE) to maintain the original format of the data
+ * so applications or business intelligence tools do not have to change in order
+ * to use these columns.
+*
+* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
+* for all possible data sizes and combinations of encryption algorithms and IV, etc.
+* Was tested with CM 2.14 & CADP CADP 8.15.0.001
+* For more information on CADP see link below.
+https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html
+* For more information on Snowflake External Functions see link below.
+https://docs.snowflake.com/en/sql-reference/external-functions-creating-aws
+
+Notes: This example uses the CADP bulk API.
+Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data
+element has Unicode characters then the supported size limit would be 1750 characters
+
+Size of spec array should be same as the number of elements if user wants to use separate spec values
+for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data
+index.
+ *
+ * @author mwarner
+ */
+
+public class ThalesAWSSnowCRDPFPEBulkUDFTester implements RequestStreamHandler {
+
+ private static int BATCHLIMIT = 10000;
+ private static final String BADDATATAG = new String("9999999999999999");
+ private static final String REVEALRETURNTAG = new String("data");
+ private static final String PROTECTRETURNTAG = new String("protected_data");
+
+ // private static final Logger logger =
+ // Logger.getLogger(LambdaRequestStreamHandlerNbrEncyrptBulkFPE.class.getName());
+ private static final Gson gson = new Gson();
+
+ public static void main(String[] args) throws Exception {
+ ThalesAWSSnowCRDPFPEBulkUDFTester nw2 = new ThalesAWSSnowCRDPFPEBulkUDFTester();
+ String request = "{\r\n" + " \"resource\": \"/snowflake_proxy\",\r\n" + " \"path\": \"/snowflake_proxy\",\r\n"
+ + " \"httpMethod\": \"POST\",\r\n" + " \"headers\": {\r\n" + " \"Accept\": \"*/*\",\r\n"
+ + " \"Accept-Encoding\": \"gzip\",\r\n" + " \"Content-Encoding\": \"gzip\",\r\n"
+ + " \"Content-Type\": \"application/json\",\r\n"
+ + " \"Host\": \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\",\r\n"
+ + " \"sf-external-function-current-query-id\": \"01b02bdf-0001-7b58-0003-27d60056b5a6\",\r\n"
+ + " \"sf-external-function-format\": \"json\",\r\n"
+ + " \"sf-external-function-format-version\": \"1.0\",\r\n"
+ + " \"sf-external-function-name\": \"thales-aws-lambda-snow-cadp-encrypt-nbr\",\r\n"
+ + " \"sf-external-function-name-base64\": \"VEhBTEVTX0NBRFBfQVdTX1RPS0VOSVpFQ0hBUg==\",\r\n"
+ + " \"sf-external-function-query-batch-id\": \"01b02bdf-0001-7b58-0003-27d60056b5a6:1:1:0:0\",\r\n"
+ + " \"sf-external-function-return-type\": \"VARIANT\",\r\n"
+ + " \"sf-external-function-return-type-base64\": \"VkFSSUFOVA==\",\r\n"
+ + " \"sf-external-function-signature\": \"(B VARCHAR)\",\r\n"
+ + " \"sf-external-function-signature-base64\": \"KEIgVkFSQ0hBUik=\",\r\n"
+ + " \"User-Agent\": \"snowflake/1.0\",\r\n"
+ + " \"x-amz-content-sha256\": \"550cead700c849cd6f159aca9dd81083306bb5c919939c1e2c4f3f909f32332b\",\r\n"
+ + " \"x-amz-date\": \"20231107T142353Z\",\r\n"
+ + " \"x-amz-security-token\": \"IQoJb3JpZ2luX2VjEG8aCXVzLWVhc3QtMSJGMEQCICKJkvz9H9bTmSg3rRMNQYHnGeNzDyseSK3GxkGkxH6NAiBm/1kqxf5r25Z7154EYa93MaMOFIdMKS/b7MheulrJxCqHAwio//////////8BEAAaDDQ5NjQ4NDM0Mjc2NCIMCJwVrXJkl8t82Se4KtsC1mBjS4CLPcQEHJ+qRnw5HHbMTWx1DkJ6SeAjMWj/QypGy3yipoi20gY49xkbo4A0kzv7bty06Oa/VPp4id/hWdVEPVypuUQpyHKYdJAn/ZrnhvyqOMlKpGSg3157v1zPduOQOz9XvjpWC12NCaF3HAcj2/aH0DfCRacsAS+XwIWxNFOtrGbsbibrSJqFe2NqbvIbLU39cVLCD51cTMRMTXhdcRpU9JhXc8+I8+eWwsgpl823OrjwTwqDCnYs/9AY+yRCUDVjuLbKKQ+lXNWlaOMD6YfBzgNnjZZDbctx1lSMHG7HRHQrFIuZe0oKnVkbeF05CSxLBorfigKX8AF/Kyq5OhSnJPdTZKG6PbtWAub+JHaelZdEo3wDTMN04MkL+Nef+gV+g2vxejZiOc+v0HL0qXB1z2OLDqXfMm1zoACue3sYLqeK2jRjWqpEoPes7rU7jHOgkrWDgfkw7ZCpqgY6ngGFyKE++chqS6vGrZJ2e/HQL12UVqyxfT2NDRQZDA0gZmWnK5drpNJECXH9DeFcVqUnnDU77Ui5v6tXfUrq9cSOOYG68b2qCYiQ+TNkQPiMZNJmx4FYDL46RWtcxyvAMqaQz/5bhwf9W/Iw2QzYuSWj4FTC29H0eD1B5Frnq69Eg3A1A1bJ8m0LfDhT7gfK/rp7I2N0LSnYbhi2ksZU6A==\",\r\n"
+ + " \"X-Amzn-Trace-Id\": \"Root=1-654a4879-4aa55d5e4a5648a0268543d1\",\r\n"
+ + " \"X-Forwarded-For\": \"3.33.33.35\",\r\n" + " \"X-Forwarded-Port\": \"443\",\r\n"
+ + " \"X-Forwarded-Proto\": \"https\"\r\n" + " },\r\n" + " \"multiValueHeaders\": {\r\n"
+ + " \"Accept\": [\r\n" + " \"*/*\"\r\n" + " ],\r\n" + " \"Accept-Encoding\": [\r\n"
+ + " \"gzip\"\r\n" + " ],\r\n" + " \"Content-Encoding\": [\r\n" + " \"gzip\"\r\n"
+ + " ],\r\n" + " \"Content-Type\": [\r\n" + " \"application/json\"\r\n" + " ],\r\n"
+ + " \"Host\": [\r\n" + " \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-current-query-id\": [\r\n" + " \"sdfsdfsdff-27d60056b5a6\"\r\n"
+ + " ],\r\n" + " \"sf-external-function-format\": [\r\n" + " \"json\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-format-version\": [\r\n" + " \"1.0\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-name\": [\r\n" + " \"THALES_CADP_AWS_R\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-name-base64\": [\r\n" + " \"wewewewewewe==\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-query-batch-id\": [\r\n"
+ + " \"01b02bdf-0001-7b58-0003-27d60056b5a6:1:1:0:0\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-return-type\": [\r\n" + " \"VARIANT\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-return-type-base64\": [\r\n" + " \"VkFSSUFOVA==\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-signature\": [\r\n" + " \"(B VARCHAR)\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-signature-base64\": [\r\n" + " \"KEIgVkFSQ0hBUik=\"\r\n"
+ + " ],\r\n" + " \"User-Agent\": [\r\n" + " \"snowflake/1.0\"\r\n" + " ],\r\n"
+ + " \"x-amz-content-sha256\": [\r\n" + " \"550cead70rtrtrtrtb5c919939c1e2c4f3f909f32332b\"\r\n"
+ + " ],\r\n" + " \"x-amz-date\": [\r\n" + " \"20231107T142353Z\"\r\n" + " ],\r\n"
+ + " \"x-amz-security-token\": [\r\n"
+ + " \"IQoJb3JpZ2luX2VjEG8aCXVzLWVhc3QtMSJGMEQCICKJkvz9H9bTmSg3rRMNQYHnGeNzDyseSK3GxkGkxH6NAiBm/1kqxf5r25Z7154EYa93MaMOFIdMKS/b7MheulrJxCqHAwio//////////8BEAAaDDQ5NjQ4NDM0Mjc2NCIMCJwVrXJkl8t82Se4KtsC1mBjS4CLPcQEHJ+qRnw5HHbMTWx1DkJ6SeAjMWj/QypGy3yipoi20gY49xkbo4A0kzv7bty06Oa/VPp4id/hWdVEPVypuUQpyHKYdJAn/ZrnhvyqOMlKpGSg3157v1zPduOQOz9XvjpWC12NCaF3HAcj2/aH0DfCRacsAS+XwIWxNFOtrGbsbibrSJqFe2NqbvIbLU39cVLCD51cTMRMTXhdcRpU9JhXc8+I8+eWwsgpl823OrjwTwqDCnYs/9AY+yRCUDVjuLbKKQ+lXNWlaOMD6YfBzgNnjZZDbctx1lSMHG7HRHQrFIuZe0oKnVkbeF05CSxLBorfigKX8AF/Kyq5OhSnJPdTZKG6PbtWAub+JHaelZdEo3wDTMN04MkL+Nef+gV+g2vxejZiOc+v0HL0qXB1z2OLDqXfMm1zoACue3sYLqeK2jRjWqpEoPes7rU7jHOgkrWDgfkw7ZCpqgY6ngGFyKE++chqS6vGrZJ2e/HQL12UVqyxfT2NDRQZDA0gZmWnK5drpNJECXH9DeFcVqUnnDU77Ui5v6tXfUrq9cSOOYG68b2qCYiQ+TNkQPiMZNJmx4FYDL46RWtcxyvAMqaQz/5bhwf9W/Iw2QzYuSWj4FTC29H0eD1B5Frnq69Eg3A1A1bJ8m0LfDhT7gfK/rp7I2N0LSnYbhi2ksZU6A==\"\r\n"
+ + " ],\r\n" + " \"X-Amzn-Trace-Id\": [\r\n"
+ + " \"Root=1-654a4879-4aa55d5e4a5648a0268543d1\"\r\n" + " ],\r\n"
+ + " \"X-Forwarded-For\": [\r\n" + " \"3.34.44.35\"\r\n" + " ],\r\n"
+ + " \"X-Forwarded-Port\": [\r\n" + " \"443\"\r\n" + " ],\r\n"
+ + " \"X-Forwarded-Proto\": [\r\n" + " \"https\"\r\n" + " ]\r\n" + " },\r\n"
+ + " \"queryStringParameters\": null,\r\n" + " \"multiValueQueryStringParameters\": null,\r\n"
+ + " \"pathParameters\": null,\r\n" + " \"stageVariables\": null,\r\n" + " \"requestContext\": {\r\n"
+ + " \"resourceId\": \"erw8lc\",\r\n" + " \"resourcePath\": \"/snowflake_proxy\",\r\n"
+ + " \"httpMethod\": \"POST\",\r\n" + " \"extendedRequestId\": \"OCBC9FLtiYcEThQ=\",\r\n"
+ + " \"requestTime\": \"07/Nov/2023:14:23:53 +0000\",\r\n"
+ + " \"path\": \"/test/snowflake_proxy\",\r\n" + " \"accountId\": \"455555555564\",\r\n"
+ + " \"protocol\": \"HTTP/1.1\",\r\n" + " \"stage\": \"test\",\r\n"
+ + " \"domainPrefix\": \"asdfsdfsfff\",\r\n" + " \"requestTimeEpoch\": 1699367033023,\r\n"
+ + " \"requestId\": \"f53460af-d302-4e3a-b837-f5a9785badcb\",\r\n" + " \"identity\": {\r\n"
+ + " \"cognitoIdentityPoolId\": null,\r\n" + " \"accountId\": \"34343434\",\r\n"
+ + " \"cognitoIdentityId\": null,\r\n" + " \"caller\": \"ARODFDFDFFDFDFX7G:snowflake\",\r\n"
+ + " \"sourceIp\": \"3.33.44.35\",\r\n" + " \"principalOrgId\": null,\r\n"
+ + " \"accessKey\": \"FAKE\",\r\n" + " \"cognitoAuthenticationType\": null,\r\n"
+ + " \"cognitoAuthenticationProvider\": null,\r\n"
+ + " \"userArn\": \"arn:aws:sts::496484342764:assumed-role/snowflakerole/snowflake\",\r\n"
+ + " \"userAgent\": \"snowflake/1.0\",\r\n"
+ + " \"user\": \"shawnscanlan@snowflakecomputing.com\"\r\n" + " },\r\n"
+ + " \"domainName\": \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\",\r\n"
+ + " \"apiId\": \"asdfsdfsfff\"\r\n" + " },\r\n"
+ + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"47764370337599674\\\"]\\n,[1,\\\"9500405729277875\\\"]\\n,[2,\\\"57\\\"]\\n,[3,\\\"29330926862189\\\"]\\n,[4,\\\"909554854973\\\"]\\n]\\n}\",\r\n"
+ + " \"isBase64Encoded\": false\r\n" + "}";
+
+//+ " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4545554545675\\\"]\\n,[1,\\\"124313145756\\\"]\\n,[2,\\\"584785473839\\\"]\\n,[3,\\\"32323232323232\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n"
+//+ " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"47764370337599674\\\"]\\n,[1,\\\"9500405729277875\\\"]\\n,[2,\\\"57\\\"]\\n,[3,\\\"64310756511368\\\"]\\n,[4,\\\"909554854973\\\"]\\n]\\n}\",\r\n"
+ // + " \"body\":
+ // \"{\\n\\\"data\\\":[\\n[0,\\\"6\\\"]\\n,[1,\\\"null\\\"]\\n,[2,\\\"49\\\"]\\n,[3,\\\"64310756511368\\\"]\\n,[4,\\\"294461560470\\\"]\\n]\\n}\",\r\n"
+ // temp{"data":[[0,"1676960812748"],[1,"933211143272"],[2,"505141998387"],[3,"64310756511368"],[4,"294461560470"]]}
+// reveal 1002001 temp{"data":[[0,"47764370337599674"],[1,"9500405729277875"],[2,"57"],[3,"29330926862189"],[4,"909554854973"]]}
+// + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"47764370337599674\\\"]\\n,[1,\\\"9500405729277875\\\"]\\n,[2,\\\"57\\\"]\\n,[3,\\\"29330926862189\\\"]\\n,[4,\\\"909554854973\\\"]\\n]\\n}\",\r\n"
+
+// + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4\\\"]\\n,[1,\\\"12431-3145-756\\\"]\\n,[2,\\\"null\\\"]\\n,[3,\\\"\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n"
+ // nw2.handleRequest(request, null, null);
+ // backup
+
+ // + " \"body\":
+ // \"{\\n\\\"data\\\":[\\n[0,\\\"454-55545-45675\\\"]\\n,[1,\\\"1243-131457-56\\\"]\\n,[2,\\\"584785473839\\\"]\\n,[3,\\\"32323232323232\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n"
+ // + " \"body\":
+ // \"{\\n\\\"data\\\":[\\n[0,\\\"9500405729277875\\\"]\\n,[1,\\\"9500405729277875\\\"]\\n,[2,\\\"49\\\"]\\n,[3,\\\"64310756511368\\\"]\\n,[4,\\\"294461560470\\\"]\\n]\\n}\",\r\n"
+ // + " \"body\":
+ // \"{\\n\\\"data\\\":[\\n[0,\\\"0\\\"]\\n,[1,\\\"null\\\"]\\n,[2,\\\"45\\\"]\\n,[3,\\\"32323232323232\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n"
+ // temp{"data":[[0,"9500405729277875"],[1,"9500405729277875"],[2,"49"],[3,"64310756511368"],[4,"294461560470"]]}
+ nw2.handleRequest2(request, null, null);
+ }
+
+ public void handleRequest2(String input, OutputStream outputStream, Context context) throws IOException {
+ // context.getLogger().log("Input: " + inputStream);
+ //String input = IOUtils.toString(inputStream, "UTF-8");
+
+ Map snowErrorMap = new HashMap();
+ String encdata = "";
+ int error_count = 0;
+ int statusCode = 200;
+
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+ JsonArray innerDataArray = new JsonArray();
+ String snowflakereturnstring = null;
+ JsonObject body_input_request = null;
+
+ int numberofchunks = 0;
+
+ String callerStr = null;
+
+ JsonObject snowflakeinput = null;
+
+ JsonArray snowflakedata = null;
+
+ String keyName = "testfaas";
+ String crdpip = System.getenv("CRDPIP");
+ // String keyName = System.getenv("CMKEYNAME");
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned
+ // when the user above does not have access to the key and if doing a
+ // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no
+ // then an error will be returned to the query, else the results set will provide ciphertext.
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ // yes,no
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB
+ // yes,no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be
+ // a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ String keymetadatalocation = System.getenv("keymetadatalocation");
+ String external_version_from_ext_source = System.getenv("keymetadata");
+ String protection_profile = System.getenv("protection_profile");
+ String mode = System.getenv("mode");
+ String datatype = System.getenv("datatype");
+
+ String showrevealkey = "yes";
+
+ int batchsize = Integer.parseInt(System.getenv("BATCHSIZE"));
+
+ String inputDataKey = null;
+ String outputDataKey = null;
+ String protectedData = null;
+ String externalkeymetadata = null;
+ String jsonBody = null;
+
+ String jsonTagForProtectReveal = null;
+
+ boolean bad_data = false;
+ if (mode.equals("protectbulk")) {
+ inputDataKey = "data_array";
+ outputDataKey = "protected_data_array";
+ jsonTagForProtectReveal = PROTECTRETURNTAG;
+ if (keymetadatalocation.equalsIgnoreCase("internal")) {
+ showrevealkey = System.getenv("showrevealinternalkey");
+ if (showrevealkey == null)
+ showrevealkey = "yes";
+ }
+ } else {
+ inputDataKey = "protected_data_array";
+ outputDataKey = "data_array";
+ jsonTagForProtectReveal = REVEALRETURNTAG;
+ }
+
+ boolean showrevealkeybool = showrevealkey.equalsIgnoreCase("yes");
+
+ try {
+
+ JsonElement rootNode = JsonParser.parseString(input).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ snowflakeinput = rootNode.getAsJsonObject();
+ if (snowflakeinput.isJsonObject()) {
+ // For some reason when using snowflake it adds \n and \ to quotes in json.
+ // the JsonParser.parseString(input).getAsJsonObject(); is supposed to remove
+ // all of those
+ // characters but it does not do it for snowflake json.
+ JsonElement bodyele = snowflakeinput.get("body");
+ String bodystr = bodyele.getAsString().replaceAll(System.lineSeparator(), "");
+ bodystr = bodystr.replaceAll("\\\\", "");
+ // System.out.println("bodystr after replace" + bodystr);
+ body_input_request = gson.fromJson(bodystr, JsonObject.class);
+ snowflakedata = body_input_request.getAsJsonArray("data");
+
+ JsonObject requestContext = snowflakeinput.getAsJsonObject("requestContext");
+
+ if (requestContext != null) {
+ JsonObject identity = requestContext.getAsJsonObject("identity");
+
+ if (identity != null) {
+ callerStr = identity.get("user").getAsString();
+ System.out.println("user: " + callerStr);
+ } else {
+ System.out.println("Identity not found.");
+ }
+ } else {
+ System.out.println("Request context not found.");
+ }
+
+ if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group
+
+ boolean founduserinuserset = findUserInUserSet(callerStr, userName, password, usersetID,
+ userSetLookupIP);
+ // System.out.println("Found User " + founduserinuserset);
+ if (!founduserinuserset)
+ throw new CustomException("1001, User Not in User Set", 1001);
+
+ } else {
+ usersetlookupbool = false;
+ }
+
+ } else {
+ System.out.println("eerror");
+
+ }
+ }
+
+ StringBuffer protection_policy_buff = new StringBuffer();
+ String notvalid = "notvalid";
+
+ int numberOfLines = snowflakedata.size();
+ int totalRowsLeft = numberOfLines;
+
+ if (batchsize > numberOfLines)
+ batchsize = numberOfLines;
+ if (batchsize >= BATCHLIMIT)
+ batchsize = BATCHLIMIT;
+
+ int i = 0;
+ int count = 0;
+ int totalcount = 0;
+
+ int dataIndex = 0; // assumes index from snowflake will always be sequential.
+ JsonObject crdp_payload = new JsonObject();
+ String sensitive = null;
+ JsonArray crdp_payload_array = new JsonArray();
+
+ OkHttpClient client = new OkHttpClient().newBuilder().build();
+ MediaType mediaType = MediaType.parse("application/json");
+ String urlStr = "http://" + crdpip + ":8090/v1/" + mode;
+
+ while (i < numberOfLines) {
+
+ for (int b = 0; b < batchsize && b < totalRowsLeft; b++) {
+
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ sensitive = checkValid(snowflakerow);
+ protection_profile = protection_profile.trim();
+ // Format the output
+ String formattedElement = String.format("\"protection_policy_name\" : \"%s\"", protection_profile);
+ protection_policy_buff.append(formattedElement);
+ protection_policy_buff.append(",");
+
+ if (mode.equals("protectbulk")) {
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ // System.out.println("adding null not charint or nbr");
+ sensitive = sensitive.replace("notvalid", "");
+ sensitive = BADDATATAG + sensitive;
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ // sensitive = sensitive.replace("notvalid", "");
+
+ }
+ encdata = sensitive;
+
+ }
+ crdp_payload_array.add(sensitive);
+ } else {
+ JsonObject protectedDataObject = new JsonObject();
+ protectedDataObject.addProperty("protected_data", sensitive);
+ if (keymetadatalocation.equalsIgnoreCase("external")) {
+ protectedDataObject.addProperty("external_version", external_version_from_ext_source);
+ }
+ crdp_payload_array.add(protectedDataObject);
+
+ }
+
+ if (count == batchsize - 1) {
+ crdp_payload.add(inputDataKey, crdp_payload_array);
+ String inputdataarray = null;
+ if (mode.equals("revealbulk")) {
+ crdp_payload.addProperty("username", callerStr);
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ jsonBody = protection_policy_buff.toString();
+ jsonBody = jsonBody.replaceFirst("\\{", " ");
+
+ } else {
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ inputdataarray = protection_policy_buff.toString();
+ jsonBody = inputdataarray.replace("{", " ");
+ }
+ jsonBody = "{" + jsonBody;
+
+ // System.out.println(jsonBody);
+ RequestBody body = RequestBody.create(mediaType, jsonBody);
+
+ // System.out.println(urlStr);
+ Request crdp_request = new Request.Builder().url(urlStr).method("POST", body)
+ .addHeader("Content-Type", "application/json").build();
+ Response crdp_response = client.newCall(crdp_request).execute();
+ String crdpreturnstr = null;
+ if (crdp_response.isSuccessful()) {
+ // Parse JSON response
+ String responseBody = crdp_response.body().string();
+ JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class);
+ JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey);
+
+ String status = jsonObject.get("status").getAsString();
+ int success_count = jsonObject.get("success_count").getAsInt();
+ error_count = jsonObject.get("error_count").getAsInt();
+ if (error_count > 0)
+ System.out.println("errors " + error_count);
+
+ for (JsonElement element : protectedDataArray) {
+
+ JsonObject protectedDataObject = element.getAsJsonObject();
+ if (protectedDataObject.has(jsonTagForProtectReveal)) {
+
+ protectedData = protectedDataObject.get(jsonTagForProtectReveal).getAsString();
+ // System.out.println(protectedData);
+ if (keymetadatalocation.equalsIgnoreCase("internal") && mode.equalsIgnoreCase("protectbulk") && !showrevealkeybool) {
+ if (protectedData.length()>7)
+ protectedData = protectedData.substring(7);
+ }
+
+ innerDataArray.add(dataIndex);
+ innerDataArray.add(new String(protectedData));
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+ if (mode.equals("protectbulk")) {
+ if (keymetadatalocation.equalsIgnoreCase("external")
+ && mode.equalsIgnoreCase("protectbulk")) {
+ externalkeymetadata = protectedDataObject.get("external_version")
+ .getAsString();
+ // System.out.println("Protected Data ext key metadata need to store this: "
+ // + externalkeymetadata);
+
+ }
+ }
+ } else if (protectedDataObject.has("error_message")) {
+ String errorMessage = protectedDataObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ snowErrorMap.put(i, errorMessage);
+ bad_data = true;
+ } else
+ System.out.println("unexpected json value from results: ");
+ dataIndex++;
+
+ }
+
+ crdp_payload_array = new JsonArray();
+ protection_policy_buff = new StringBuffer();
+ numberofchunks++;
+ totalcount = totalcount + count;
+ count = 0;
+ } else {// throw error....
+ System.err.println("Request failed with status code: " + crdp_response.code());
+ throw new CustomException("1010, Unexpected Error ", 1010);
+ }
+
+ } else {
+ count++;
+ }
+ totalRowsLeft--;
+ i++;
+ }
+ }
+ if (count > 0) {
+ crdp_payload.add(inputDataKey, crdp_payload_array);
+ String inputdataarray = null;
+ if (mode.equals("revealbulk")) {
+ crdp_payload.addProperty("username", callerStr);
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ jsonBody = protection_policy_buff.toString();
+ jsonBody = jsonBody.replaceFirst("\\{", " ");
+
+ } else {
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ inputdataarray = protection_policy_buff.toString();
+ jsonBody = inputdataarray.replace("{", " ");
+ }
+ jsonBody = "{" + jsonBody;
+
+ // System.out.println(jsonBody);
+ RequestBody body = RequestBody.create(mediaType, jsonBody);
+
+ // System.out.println(urlStr);
+ Request crdp_request = new Request.Builder().url(urlStr).method("POST", body)
+ .addHeader("Content-Type", "application/json").build();
+ Response crdp_response = client.newCall(crdp_request).execute();
+ String crdpreturnstr = null;
+ if (crdp_response.isSuccessful()) {
+ // Parse JSON response
+ String responseBody = crdp_response.body().string();
+ JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class);
+ JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey);
+
+ String status = jsonObject.get("status").getAsString();
+ int success_count = jsonObject.get("success_count").getAsInt();
+ error_count = jsonObject.get("error_count").getAsInt();
+ if (error_count > 0)
+ System.out.println("errors " + error_count);
+
+ if (mode.equals("protectbulk")) {
+
+ for (JsonElement element : protectedDataArray) {
+ JsonObject protectedDataObject = element.getAsJsonObject();
+ if (protectedDataObject.has("protected_data")) {
+
+ protectedData = protectedDataObject.get("protected_data").getAsString();
+
+ innerDataArray.add(dataIndex);
+ innerDataArray.add(new String(protectedData));
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ if (keymetadatalocation.equalsIgnoreCase("external")
+ && mode.equalsIgnoreCase("protectbulk")) {
+ externalkeymetadata = protectedDataObject.get("external_version").getAsString();
+ // System.out.println("Protected Data ext key metadata need to store this: "
+ // + externalkeymetadata);
+
+ }
+
+ if (keymetadatalocation.equalsIgnoreCase("internal") && mode.equalsIgnoreCase("protectbulk") && !showrevealkeybool) {
+ if (protectedData.length()>7)
+ protectedData = protectedData.substring(7);
+ }
+
+ } else if (protectedDataObject.has("error_message")) {
+ String errorMessage = protectedDataObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ snowErrorMap.put(i, errorMessage);
+ bad_data = true;
+ } else {
+ System.out.println("unexpected json value from results: ");
+ throw new CustomException("1010, Unexpected Error ", 1010);
+ }
+ dataIndex++;
+ }
+ } else {
+ // reveal logic
+
+ for (JsonElement element : protectedDataArray) {
+ JsonObject protectedDataObject = element.getAsJsonObject();
+ if (protectedDataObject.has("data")) {
+ protectedData = protectedDataObject.get("data").getAsString();
+ // System.out.println(protectedData);
+
+ innerDataArray.add(dataIndex);
+ innerDataArray.add(new String(protectedData));
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ } else if (protectedDataObject.has("error_message")) {
+ String errorMessage = protectedDataObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ snowErrorMap.put(i, errorMessage);
+ bad_data = true;
+ } else
+ System.out.println("unexpected json value from results: ");
+ dataIndex++;
+ }
+ }
+
+ crdp_response.close();
+
+ numberofchunks++;
+
+ totalcount = totalcount + count;
+ count = 0;
+
+ } else {
+ System.err.println("Request failed with status code: " + crdp_response.code());
+ }
+ }
+ System.out.println("total chuncks " + numberofchunks);
+
+ bodyObject.add("data", dataArray);
+ JsonObject inputJsonObject = new JsonObject();
+ String bodyString = bodyObject.toString();
+ inputJsonObject.addProperty("statusCode", 200);
+ inputJsonObject.addProperty("body", bodyString);
+
+ snowflakereturnstring = inputJsonObject.toString();
+
+ } catch (Exception e) {
+ System.out.println("in exception with " + e.getMessage());
+ snowflakereturnstring = "exception ";
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
+
+ bodyObject = new JsonObject();
+ dataArray = new JsonArray();
+ innerDataArray = new JsonArray();
+ int nbrofrows = snowflakedata.size();
+ for (int i = 0; i < nbrofrows; i++) {
+
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ for (int j = 0; j < snowflakerow.size(); j++) {
+ if (j == 1) {
+ // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
+ // FPE example
+ String sensitive = checkValid(snowflakerow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null")
+ || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+
+ }
+ encdata = sensitive;
+
+ } else {
+ // System.out.println("normal number data" + sensitive);
+ }
+ innerDataArray.add(sensitive);
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ int row_number = snowflakecolumn.getAsInt();
+ innerDataArray.add(row_number);
+ }
+ }
+ }
+
+ bodyObject.add("data", dataArray);
+ JsonObject inputJsonObject = new JsonObject();
+ String bodyString = bodyObject.toString();
+ inputJsonObject.addProperty("statusCode", 200);
+ inputJsonObject.addProperty("body", bodyString);
+
+ snowflakereturnstring = inputJsonObject.toString();
+ // System.out.println(" new data " + snowflakereturnstring);
+
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+
+ } finally {
+
+ }
+ System.out.println(snowflakereturnstring);
+ //outputStream.write(snowflakereturnstring.getBytes());
+
+ }
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+
+ boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
+
+ return founduserinuserset;
+
+ }
+
+ public String checkValid(JsonArray snowrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (snowrow != null && snowrow.size() > 0) {
+ JsonElement element = snowrow.get(1);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer snowflakereturndatasb = new StringBuffer();
+
+ snowflakereturndatasb.append("{ \"statusCode\":");
+ snowflakereturndatasb.append(statusCode);
+ snowflakereturndatasb.append(",");
+ snowflakereturndatasb.append(" \"body\": {");
+ snowflakereturndatasb.append(" \"data\": [");
+ snowflakereturndatasb.append("] }}");
+ System.out.println("in exception with ");
+ return snowflakereturndatasb.toString();
+ }
+
+ @Override
+ public void handleRequest(InputStream input, OutputStream output, Context context) throws IOException {
+ // TODO Auto-generated method stub
+
+ }
+
+
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCRDPFPEUDFTester.java b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCRDPFPEUDFTester.java
new file mode 100644
index 00000000..74cedb37
--- /dev/null
+++ b/database/snowflake/Thales-Snow-AWS-UDF/src/test/java/ThalesAWSSnowCRDPFPEUDFTester.java
@@ -0,0 +1,537 @@
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+
+import okhttp3.MediaType;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
+/* This sample AWS Lambda Function is used to implement a Snowflake Database User Defined Function(UDF). /*
+ * It is an example of how to use Thales Cipher Trust Application Data Protection (CADP)
+ * to protect sensitive data in a column. This example uses
+ * Format Preserve Encryption (FPE) to maintain the original format of the data
+ * so applications or business intelligence tools do not have to change in order
+ * to use these columns.
+ *
+* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
+* for all possible data sizes and combinations of encryption algorithms and IV, etc.
+* Was tested with CM 2.14 & CADP CADP 8.15.0.001
+* For more information on CADP see link below.
+https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html
+* For more information on Snowflake External Functions see link below.
+https://docs.snowflake.com/en/sql-reference/external-functions-creating-aws
+ *
+ */
+
+public class ThalesAWSSnowCRDPFPEUDFTester implements RequestStreamHandler {
+
+ private static final Gson gson = new Gson();
+ private static final String BADDATATAG = new String("9999999999999999");
+ private static final String REVEALRETURNTAG = new String("data");
+ private static final String PROTECTRETURNTAG = new String("protected_data");
+
+ public static void main(String[] args) throws Exception
+
+ {
+ ThalesAWSSnowCRDPFPEUDFTester nw2 = new ThalesAWSSnowCRDPFPEUDFTester();
+ String request = "{\r\n" + " \"resource\": \"/snowflake_proxy\",\r\n" + " \"path\": \"/snowflake_proxy\",\r\n"
+ + " \"httpMethod\": \"POST\",\r\n" + " \"headers\": {\r\n" + " \"Accept\": \"*/*\",\r\n"
+ + " \"Accept-Encoding\": \"gzip\",\r\n" + " \"Content-Encoding\": \"gzip\",\r\n"
+ + " \"Content-Type\": \"application/json\",\r\n"
+ + " \"Host\": \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\",\r\n"
+ + " \"sf-external-function-current-query-id\": \"01b02bdf-0001-7b58-0003-27d60056b5a6\",\r\n"
+ + " \"sf-external-function-format\": \"json\",\r\n"
+ + " \"sf-external-function-format-version\": \"1.0\",\r\n"
+ + " \"sf-external-function-name\": \"thales-aws-lambda-snow-cadp-encrypt-nbr\",\r\n"
+ + " \"sf-external-function-name-base64\": \"VEhBTEVTX0NBRFBfQVdTX1RPS0VOSVpFQ0hBUg==\",\r\n"
+ + " \"sf-external-function-query-batch-id\": \"01b02bdf-0001-7b58-0003-27d60056b5a6:1:1:0:0\",\r\n"
+ + " \"sf-external-function-return-type\": \"VARIANT\",\r\n"
+ + " \"sf-external-function-return-type-base64\": \"VkFSSUFOVA==\",\r\n"
+ + " \"sf-external-function-signature\": \"(B VARCHAR)\",\r\n"
+ + " \"sf-external-function-signature-base64\": \"KEIgVkFSQ0hBUik=\",\r\n"
+ + " \"User-Agent\": \"snowflake/1.0\",\r\n"
+ + " \"x-amz-content-sha256\": \"550cead700c849cd6f159aca9dd81083306bb5c919939c1e2c4f3f909f32332b\",\r\n"
+ + " \"x-amz-date\": \"20231107T142353Z\",\r\n"
+ + " \"x-amz-security-token\": \"IQoJb3JpZ2luX2VjEG8aCXVzLWVhc3QtMSJGMEQCICKJkvz9H9bTmSg3rRMNQYHnGeNzDyseSK3GxkGkxH6NAiBm/1kqxf5r25Z7154EYa93MaMOFIdMKS/b7MheulrJxCqHAwio//////////8BEAAaDDQ5NjQ4NDM0Mjc2NCIMCJwVrXJkl8t82Se4KtsC1mBjS4CLPcQEHJ+qRnw5HHbMTWx1DkJ6SeAjMWj/QypGy3yipoi20gY49xkbo4A0kzv7bty06Oa/VPp4id/hWdVEPVypuUQpyHKYdJAn/ZrnhvyqOMlKpGSg3157v1zPduOQOz9XvjpWC12NCaF3HAcj2/aH0DfCRacsAS+XwIWxNFOtrGbsbibrSJqFe2NqbvIbLU39cVLCD51cTMRMTXhdcRpU9JhXc8+I8+eWwsgpl823OrjwTwqDCnYs/9AY+yRCUDVjuLbKKQ+lXNWlaOMD6YfBzgNnjZZDbctx1lSMHG7HRHQrFIuZe0oKnVkbeF05CSxLBorfigKX8AF/Kyq5OhSnJPdTZKG6PbtWAub+JHaelZdEo3wDTMN04MkL+Nef+gV+g2vxejZiOc+v0HL0qXB1z2OLDqXfMm1zoACue3sYLqeK2jRjWqpEoPes7rU7jHOgkrWDgfkw7ZCpqgY6ngGFyKE++chqS6vGrZJ2e/HQL12UVqyxfT2NDRQZDA0gZmWnK5drpNJECXH9DeFcVqUnnDU77Ui5v6tXfUrq9cSOOYG68b2qCYiQ+TNkQPiMZNJmx4FYDL46RWtcxyvAMqaQz/5bhwf9W/Iw2QzYuSWj4FTC29H0eD1B5Frnq69Eg3A1A1bJ8m0LfDhT7gfK/rp7I2N0LSnYbhi2ksZU6A==\",\r\n"
+ + " \"X-Amzn-Trace-Id\": \"Root=1-654a4879-4aa55d5e4a5648a0268543d1\",\r\n"
+ + " \"X-Forwarded-For\": \"3.33.33.35\",\r\n" + " \"X-Forwarded-Port\": \"443\",\r\n"
+ + " \"X-Forwarded-Proto\": \"https\"\r\n" + " },\r\n" + " \"multiValueHeaders\": {\r\n"
+ + " \"Accept\": [\r\n" + " \"*/*\"\r\n" + " ],\r\n" + " \"Accept-Encoding\": [\r\n"
+ + " \"gzip\"\r\n" + " ],\r\n" + " \"Content-Encoding\": [\r\n" + " \"gzip\"\r\n"
+ + " ],\r\n" + " \"Content-Type\": [\r\n" + " \"application/json\"\r\n" + " ],\r\n"
+ + " \"Host\": [\r\n" + " \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-current-query-id\": [\r\n" + " \"sdfsdfsdff-27d60056b5a6\"\r\n"
+ + " ],\r\n" + " \"sf-external-function-format\": [\r\n" + " \"json\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-format-version\": [\r\n" + " \"1.0\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-name\": [\r\n" + " \"THALES_CADP_AWS_R\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-name-base64\": [\r\n" + " \"wewewewewewe==\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-query-batch-id\": [\r\n"
+ + " \"01b02bdf-0001-7b58-0003-27d60056b5a6:1:1:0:0\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-return-type\": [\r\n" + " \"VARIANT\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-return-type-base64\": [\r\n" + " \"VkFSSUFOVA==\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-signature\": [\r\n" + " \"(B VARCHAR)\"\r\n" + " ],\r\n"
+ + " \"sf-external-function-signature-base64\": [\r\n" + " \"KEIgVkFSQ0hBUik=\"\r\n"
+ + " ],\r\n" + " \"User-Agent\": [\r\n" + " \"snowflake/1.0\"\r\n" + " ],\r\n"
+ + " \"x-amz-content-sha256\": [\r\n" + " \"550cead70rtrtrtrtb5c919939c1e2c4f3f909f32332b\"\r\n"
+ + " ],\r\n" + " \"x-amz-date\": [\r\n" + " \"20231107T142353Z\"\r\n" + " ],\r\n"
+ + " \"x-amz-security-token\": [\r\n"
+ + " \"IQoJb3JpZ2luX2VjEG8aCXVzLWVhc3QtMSJGMEQCICKJkvz9H9bTmSg3rRMNQYHnGeNzDyseSK3GxkGkxH6NAiBm/1kqxf5r25Z7154EYa93MaMOFIdMKS/b7MheulrJxCqHAwio//////////8BEAAaDDQ5NjQ4NDM0Mjc2NCIMCJwVrXJkl8t82Se4KtsC1mBjS4CLPcQEHJ+qRnw5HHbMTWx1DkJ6SeAjMWj/QypGy3yipoi20gY49xkbo4A0kzv7bty06Oa/VPp4id/hWdVEPVypuUQpyHKYdJAn/ZrnhvyqOMlKpGSg3157v1zPduOQOz9XvjpWC12NCaF3HAcj2/aH0DfCRacsAS+XwIWxNFOtrGbsbibrSJqFe2NqbvIbLU39cVLCD51cTMRMTXhdcRpU9JhXc8+I8+eWwsgpl823OrjwTwqDCnYs/9AY+yRCUDVjuLbKKQ+lXNWlaOMD6YfBzgNnjZZDbctx1lSMHG7HRHQrFIuZe0oKnVkbeF05CSxLBorfigKX8AF/Kyq5OhSnJPdTZKG6PbtWAub+JHaelZdEo3wDTMN04MkL+Nef+gV+g2vxejZiOc+v0HL0qXB1z2OLDqXfMm1zoACue3sYLqeK2jRjWqpEoPes7rU7jHOgkrWDgfkw7ZCpqgY6ngGFyKE++chqS6vGrZJ2e/HQL12UVqyxfT2NDRQZDA0gZmWnK5drpNJECXH9DeFcVqUnnDU77Ui5v6tXfUrq9cSOOYG68b2qCYiQ+TNkQPiMZNJmx4FYDL46RWtcxyvAMqaQz/5bhwf9W/Iw2QzYuSWj4FTC29H0eD1B5Frnq69Eg3A1A1bJ8m0LfDhT7gfK/rp7I2N0LSnYbhi2ksZU6A==\"\r\n"
+ + " ],\r\n" + " \"X-Amzn-Trace-Id\": [\r\n"
+ + " \"Root=1-654a4879-4aa55d5e4a5648a0268543d1\"\r\n" + " ],\r\n"
+ + " \"X-Forwarded-For\": [\r\n" + " \"3.34.44.35\"\r\n" + " ],\r\n"
+ + " \"X-Forwarded-Port\": [\r\n" + " \"443\"\r\n" + " ],\r\n"
+ + " \"X-Forwarded-Proto\": [\r\n" + " \"https\"\r\n" + " ]\r\n" + " },\r\n"
+ + " \"queryStringParameters\": null,\r\n" + " \"multiValueQueryStringParameters\": null,\r\n"
+ + " \"pathParameters\": null,\r\n" + " \"stageVariables\": null,\r\n" + " \"requestContext\": {\r\n"
+ + " \"resourceId\": \"erw8lc\",\r\n" + " \"resourcePath\": \"/snowflake_proxy\",\r\n"
+ + " \"httpMethod\": \"POST\",\r\n" + " \"extendedRequestId\": \"OCBC9FLtiYcEThQ=\",\r\n"
+ + " \"requestTime\": \"07/Nov/2023:14:23:53 +0000\",\r\n"
+ + " \"path\": \"/test/snowflake_proxy\",\r\n" + " \"accountId\": \"455555555564\",\r\n"
+ + " \"protocol\": \"HTTP/1.1\",\r\n" + " \"stage\": \"test\",\r\n"
+ + " \"domainPrefix\": \"asdfsdfsfff\",\r\n" + " \"requestTimeEpoch\": 1699367033023,\r\n"
+ + " \"requestId\": \"f53460af-d302-4e3a-b837-f5a9785badcb\",\r\n" + " \"identity\": {\r\n"
+ + " \"cognitoIdentityPoolId\": null,\r\n" + " \"accountId\": \"34343434\",\r\n"
+ + " \"cognitoIdentityId\": null,\r\n" + " \"caller\": \"ARODFDFDFFDFDFX7G:snowflake\",\r\n"
+ + " \"sourceIp\": \"3.33.44.35\",\r\n" + " \"principalOrgId\": null,\r\n"
+ + " \"accessKey\": \"FAKE\",\r\n" + " \"cognitoAuthenticationType\": null,\r\n"
+ + " \"cognitoAuthenticationProvider\": null,\r\n"
+ + " \"userArn\": \"arn:aws:sts::496484342764:assumed-role/snowflakerole/snowflake\",\r\n"
+ + " \"userAgent\": \"snowflake/1.0\",\r\n"
+ + " \"user\": \"AROAXHGGF3PWF4RUKPX7G:snowflake\"\r\n" + " },\r\n"
+ + " \"domainName\": \"asdfsdfsfff.execute-api.us-east-2.amazonaws.com\",\r\n"
+ + " \"apiId\": \"asdfsdfsfff\"\r\n" + " },\r\n"
+ + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"100300150\\\"]\\n]\\n}\",\r\n" + " \"isBase64Encoded\": false\r\n"
+ + "}";
+
+ // "body":"{\"data\":[[0,\"100300150\"]]}"}
+
+ // + " \"body\":
+ // \"{\\n\\\"data\\\":[\\n[0,\\\"4545554545675\\\"]\\n,[1,\\\"124313145756\\\"]\\n,[2,\\\"584785473839\\\"]\\n,[3,\\\"32323232323232\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n"
+ // + " \"body\":
+ // \"{\\n\\\"data\\\":[\\n[0,\\\"47764370337599674\\\"]\\n,[1,\\\"9500405729277875\\\"]\\n,[2,\\\"57\\\"]\\n,[3,\\\"64310756511368\\\"]\\n,[4,\\\"909554854973\\\"]\\n]\\n}\",\r\n"
+ // + " \"body\":
+ // \"{\\n\\\"data\\\":[\\n[0,\\\"6\\\"]\\n,[1,\\\"null\\\"]\\n,[2,\\\"49\\\"]\\n,[3,\\\"64310756511368\\\"]\\n,[4,\\\"294461560470\\\"]\\n]\\n}\",\r\n"
+ // temp{"data":[[0,"1676960812748"],[1,"933211143272"],[2,"505141998387"],[3,"64310756511368"],[4,"294461560470"]]}
+ // reveal 1002001
+ // temp{"data":[[0,"47764370337599674"],[1,"9500405729277875"],[2,"57"],[3,"29330926862189"],[4,"909554854973"]]}
+// + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"47764370337599674\\\"]\\n,[1,\\\"9500405729277875\\\"]\\n,[2,\\\"57\\\"]\\n,[3,\\\"29330926862189\\\"]\\n,[4,\\\"909554854973\\\"]\\n]\\n}\",\r\n"
+
+// + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4\\\"]\\n,[1,\\\"12431-3145-756\\\"]\\n,[2,\\\"null\\\"]\\n,[3,\\\"\\\"]\\n,[4,\\\"121212121212\\\"]\\n]\\n}\",\r\n"
+
+ // + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4545554545675\\\"]\\n]\\n}\",\r\n"
+// + " \"body\": \"{\\n\\\"data\\\":[\\n[0,\\\"4545554-5456-75\\\"]\\n]\\n}\",\r\n"
+ // "body":
+ // "{\n\"data\":[\n[0,\"ndorgan5@sf_tuts.com\"],\n[1,\"cdevereu6@sf_tuts.co.au\"],\n[2,\"gglaserman7@sf_tuts.com\"],\n[3,\"icasemore8@sf_tuts.com\"],\n[4,\"chovie9@sf_tuts.com\"],\n[5,\"afeatherstona@sf_tuts.com\"],\n[6,\"hanneslieb@sf_tuts.com\"],\n[7,\"bciccoc@sf_tuts.com\"],\n[8,\"bdurnalld@sf_tuts.com\"],\n[9,\"kmacconnale@sf_tuts.com\"],\n[10,\"wsizeyf@sf_tuts.com\"],\n[11,\"dmcgowrang@sf_tuts.com\"],\n[12,\"cbedderh@sf_tuts.co.au\"],\n[13,\"davoryi@sf_tuts.com\"],\n[14,\"rtalmadgej@sf_tuts.co.uk\"],\n[15,\"lboissier@sf_tuts.com\"],\n[16,\"ihanks1@sf_tuts.com\"],\n[17,\"alaudham2@sf_tuts.com\"],\n[18,\"ecornner3@sf_tuts.com\"],\n[19,\"hgoolding4@sf_tuts.com\"],\n[20,\"adavidovitsk@sf_tuts.com\"],\n[21,\"vshermorel@sf_tuts.com\"],\n[22,\"rmattysm@sf_tuts.com\"],\n[23,\"soluwatoyinn@sf_tuts.com\"],\n[24,\"gbassfordo@sf_tuts.co.uk\"]\n]\n}",
+ // "isBase64Encoded": false
+
+ // " \"user\": \"ADFDFDFFG:snowflake\"\r\n" +
+ // + " \"user\": \"shawnscanlan@snowflakecomputing.com\"\r\n" + " },\r\n"
+ nw2.handleRequest(request, null, null);
+
+ }
+
+ public void handleRequest(String inputStream, OutputStream outputStream, Context context) throws Exception {
+ Map bqErrorMap = new HashMap();
+ String encdata = "";
+ String snowflakereturnstring = null;
+
+ String input = inputStream;
+ JsonObject snowflakebody = null;
+ int statusCode = 200;
+
+ // https://www.baeldung.com/java-aws-lambda
+ JsonObject snowflakeinput = null;
+ String callerStr = null;
+ JsonArray snowflakedata = null;
+
+ String keyName = "testfaas";
+ String crdpip = System.getenv("CRDPIP");
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be
+ // returned when the user above does not have access to the key and if doing a lookup in the userset
+ // and the user does not exist. If returnciphertextforuserwithnokeyaccess = no then an error will be
+ // returned to the query, else the results set will provide ciphertext.
+ // yes/no
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB?
+ // yes/no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it
+ // is the userset in CM but could be a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ // keymetadatalocation keymeta data can be internal or external
+ String keymetadatalocation = System.getenv("keymetadatalocation");
+ // keymetadata represents a 7 digit value that contains policy version and key version. example 1001001.
+ // Normally would come from a database
+ String external_version_from_ext_source = System.getenv("keymetadata");
+ // protection_profile = the protection profile to be used for protect or reveal. This is in CM under application
+ // data protection/protection profiles.
+ String protection_profile = System.getenv("protection_profile");
+ // mode of operation. valid values are protect/reveal
+ String mode = System.getenv("mode");
+ String datatype = System.getenv("datatype");
+
+ String showrevealkey = "yes";
+
+ String dataKey = null;
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+ JsonArray innerDataArray = new JsonArray();
+ String jsonTagForProtectReveal = null;
+
+ boolean bad_data = false;
+
+ if (mode.equals("protect")) {
+ dataKey = "data";
+ jsonTagForProtectReveal = PROTECTRETURNTAG;
+ if (keymetadatalocation.equalsIgnoreCase("internal")) {
+ showrevealkey = System.getenv("showrevealinternalkey");
+ if (showrevealkey == null)
+ showrevealkey = "yes";
+ }
+ } else {
+ dataKey = "protected_data";
+ jsonTagForProtectReveal = REVEALRETURNTAG;
+ }
+
+ boolean showrevealkeybool = showrevealkey.equalsIgnoreCase("yes");
+
+ try {
+
+ JsonElement rootNode = JsonParser.parseString(input).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ snowflakeinput = rootNode.getAsJsonObject();
+ if (snowflakeinput.isJsonObject()) {
+ // For some reason when using snowflake it adds \n and \ to quotes in json.
+ // the JsonParser.parseString(input).getAsJsonObject(); is supposed to remove
+ // all of those
+ // characters but it does not do it for snowflake json.
+ JsonElement bodyele = snowflakeinput.get("body");
+ String bodystr = bodyele.getAsString().replaceAll(System.lineSeparator(), "");
+ // System.out.println("bodystr before replace" + bodystr );
+ bodystr = bodystr.replaceAll("\\\\", "");
+ // System.out.println("bodystr after replace" + bodystr );
+ snowflakebody = gson.fromJson(bodystr, JsonObject.class);
+ snowflakedata = snowflakebody.getAsJsonArray("data");
+ JsonObject requestContext = snowflakeinput.getAsJsonObject("requestContext");
+
+ if (requestContext != null) {
+ JsonObject identity = requestContext.getAsJsonObject("identity");
+
+ if (identity != null) {
+ callerStr = identity.get("user").getAsString();
+ // String[] parts = callerStr.split(":");
+ // callerStr = parts[0];
+
+ System.out.println("user: " + callerStr);
+ } else {
+ System.out.println("Identity not found.");
+ }
+ } else {
+ System.out.println("Request context not found.");
+ }
+
+ if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group
+
+ boolean founduserinuserset = findUserInUserSet(callerStr, userName, password, usersetID,
+ userSetLookupIP);
+ // System.out.println("Found User " + founduserinuserset);
+ if (!founduserinuserset)
+ throw new CustomException("1001, User Not in User Set", 1001);
+
+ } else {
+ usersetlookupbool = false;
+ }
+
+ } else {
+ System.out.println("eerror");
+
+ }
+ }
+
+ // Serialization
+
+ String protectedData = null;
+ String externalkeymetadata = null;
+ String crdpjsonBody = null;
+ // String external_version_from_ext_source = "1004001";
+ // String external_version_from_ext_source = "1001001";
+ int row_number = 0;
+ JsonObject crdp_payload = new JsonObject();
+
+ crdp_payload.addProperty("protection_policy_name", protection_profile);
+ String sensitive = null;
+ OkHttpClient client = new OkHttpClient().newBuilder().build();
+ MediaType mediaType = MediaType.parse("application/json");
+ String urlStr = "http://" + crdpip + ":8090/v1/" + mode;
+ int nbrofrows = snowflakedata.size();
+
+ for (int i = 0; i < nbrofrows; i++) {
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ for (int j = 0; j < snowflakerow.size(); j++) {
+
+ if (j == 1) {
+ // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
+ // FPE example
+ sensitive = checkValid(snowflakerow);
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+
+ }
+ encdata = sensitive;
+
+ } else {
+
+ crdp_payload.addProperty(dataKey, sensitive);
+ if (mode.equals("reveal")) {
+ crdp_payload.addProperty("username", callerStr);
+ if (keymetadatalocation.equalsIgnoreCase("external")) {
+ crdp_payload.addProperty("external_version", external_version_from_ext_source);
+ }
+ }
+ crdpjsonBody = crdp_payload.toString();
+ System.out.println(crdpjsonBody);
+ RequestBody body = RequestBody.create(mediaType, crdpjsonBody);
+
+ // String urlStr = "\"http://" + cmip + ":8090/v1/" + mode+ "\"";
+ System.out.println(urlStr);
+ Request crdp_request = new Request.Builder()
+ // .url("http://192.168.159.143:8090/v1/protect").method("POST", body)
+ .url(urlStr).method("POST", body).addHeader("Content-Type", "application/json")
+ .build();
+ Response crdp_response = client.newCall(crdp_request).execute();
+
+ if (crdp_response.isSuccessful()) {
+
+ // Parse JSON response
+ String responseBody = crdp_response.body().string();
+ Gson gson = new Gson();
+ JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class);
+
+ if (jsonObject.has(jsonTagForProtectReveal)) {
+ protectedData = jsonObject.get(jsonTagForProtectReveal).getAsString();
+ if (keymetadatalocation.equalsIgnoreCase("external")
+ && mode.equalsIgnoreCase("protect")) {
+ externalkeymetadata = jsonObject.get("external_version").getAsString();
+ System.out.println("Protected Data ext key metadata need to store this: "
+ + externalkeymetadata);
+ }
+
+ if (keymetadatalocation.equalsIgnoreCase("internal")
+ && mode.equalsIgnoreCase("protect") && !showrevealkeybool) {
+ if (protectedData.length() > 7)
+ protectedData = protectedData.substring(7);
+ }
+
+ } else if (jsonObject.has("error_message")) {
+ String errorMessage = jsonObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ bqErrorMap.put(i, errorMessage);
+ bad_data = true;
+ } else
+ System.out.println("unexpected json value from results: ");
+ } else {
+ System.err.println("Request failed with status code: " + crdp_response.code());
+ }
+
+ crdp_response.close();
+//test to see if work for just 5
+ encdata = protectedData;
+
+ }
+
+ innerDataArray.add(encdata);
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ row_number = snowflakecolumn.getAsInt();
+ innerDataArray.add(row_number);
+ }
+
+ }
+
+ }
+
+ bodyObject.add("data", dataArray);
+ JsonObject inputJsonObject = new JsonObject();
+ String bodyString = bodyObject.toString();
+ inputJsonObject.addProperty("statusCode", 200);
+ inputJsonObject.addProperty("body", bodyString);
+
+ snowflakereturnstring = inputJsonObject.toString();
+ System.out.println(" new data " + snowflakereturnstring);
+
+ } catch (Exception e) {
+
+ System.out.println("in exception with " + e.getMessage());
+ snowflakereturnstring = "exception ";
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
+
+ bodyObject = new JsonObject();
+ dataArray = new JsonArray();
+ innerDataArray = new JsonArray();
+ int nbrofrows = snowflakedata.size();
+ for (int i = 0; i < nbrofrows; i++) {
+
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ for (int j = 0; j < snowflakerow.size(); j++) {
+ if (j == 1) {
+ // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
+ // FPE example
+ String sensitive = checkValid(snowflakerow);
+
+ sensitive = checkValid(snowflakerow);
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ System.out.println("adding null not charint or nbr");
+ sensitive = sensitive.replace("notvalid", "");
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null")
+ || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+
+ }
+ encdata = sensitive;
+
+ } else {
+ System.out.println("normal number data" + sensitive);
+ }
+ innerDataArray.add(sensitive);
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ int row_number = snowflakecolumn.getAsInt();
+ innerDataArray.add(row_number);
+ }
+ }
+ }
+
+ bodyObject.add("data", dataArray);
+ JsonObject inputJsonObject = new JsonObject();
+ String bodyString = bodyObject.toString();
+ inputJsonObject.addProperty("statusCode", 200);
+ inputJsonObject.addProperty("body", bodyString);
+
+ snowflakereturnstring = inputJsonObject.toString();
+ System.out.println(" new data " + snowflakereturnstring);
+
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+
+ } finally {
+
+ }
+ System.out.println("results" + snowflakereturnstring);
+ // outputStream.write(snowflakereturnstring.getBytes());
+
+ }
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer snowflakereturndatasb = new StringBuffer();
+
+ snowflakereturndatasb.append("{ \"statusCode\":");
+ snowflakereturndatasb.append(statusCode);
+ snowflakereturndatasb.append(",");
+ snowflakereturndatasb.append(" \"body\": {");
+ snowflakereturndatasb.append(" \"data\": [");
+ snowflakereturndatasb.append("] }}");
+ System.out.println("in exception with ");
+ return snowflakereturndatasb.toString();
+ }
+
+ public String checkValid(JsonArray snowrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (snowrow != null && snowrow.size() > 0) {
+ JsonElement element = snowrow.get(1);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ @Override
+ public void handleRequest(InputStream input, OutputStream output, Context context) throws IOException {
+ // TODO Auto-generated method stub
+
+ }
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+
+ boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
+
+ return founduserinuserset;
+
+ }
+
+}
\ No newline at end of file
diff --git a/database/snowflake/CADP-SNOW-GCP-Functions/pom.xml b/database/snowflake/Thales-Snow-GCP-UDF/pom.xml
similarity index 88%
rename from database/snowflake/CADP-SNOW-GCP-Functions/pom.xml
rename to database/snowflake/Thales-Snow-GCP-UDF/pom.xml
index 38ca42c9..578df43b 100644
--- a/database/snowflake/CADP-SNOW-GCP-Functions/pom.xml
+++ b/database/snowflake/Thales-Snow-GCP-UDF/pom.xml
@@ -1,8 +1,11 @@
+
+
+
4.0.0
Thales
- CADP-SNOW-GCP-Tokenize
- 0.0.1-SNAPSHOT
+ Thales-Snow-GCP-UDF
+ 0.0.9-SNAPSHOT
11
11
@@ -11,25 +14,29 @@
1.70
1.10
- 32.0.0-jre
+ 31.1-jre
2.9.0
2.17.2
2.17.2
.000
- CADP-GCP-Function
+ Thales-Snow-GCP-UDF
io.github.thalescpl-io.cadp
CADP_for_JAVA
- 8.13.1.000
+ 8.16.0.000
org.apache.commons
commons-lang3
3.12.0
-
+
+ com.squareup.okhttp3
+ okhttp
+ 4.10.0
+
commons-codec
commons-codec
@@ -96,11 +103,6 @@
log4j-core
${log4j-core.version}
-
- IngrianNAE
- IngrianNAE
- 8.12.6.000
-
@@ -129,14 +131,6 @@
-
-
- org.apache.maven.plugins
- maven-install-plugin
- 3.0.1
-
-
-
-
\ No newline at end of file
+
diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/CMUserSetHelper.java b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/CMUserSetHelper.java
new file mode 100644
index 00000000..06f17015
--- /dev/null
+++ b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/CMUserSetHelper.java
@@ -0,0 +1,503 @@
+package com.example;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.security.KeyStore;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509TrustManager;
+/*
+ * This app provides a number of different helper methods dealing with CM Application Data Protection UserSets. There is a method to find
+ * a user in a userset and another method to populate the userset from a flat file. Usersets are typically used within
+ * an access policy but they are not restricted to that usage.
+ * Was tested with CM 2.14
+ * Note: This source code is only to be used for testing and proof of concepts.
+ * @author mwarner
+ *
+ */
+
+public class CMUserSetHelper {
+
+ static String hostnamevalidate = "yourhostname";
+
+ String usersetid = "716f01a6-5cab-4799-925a-6dc2d8712fc1";
+ String cmIP = "yourip";
+ String apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?name=";
+ // + username_in_userset;
+ String addusertouserset = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users";
+
+ int totalrecords = 0;
+ String authUrl = "https://" + cmIP + "/api/v1/auth/tokens";
+
+ static boolean debug = true;
+ int chunksize = 5;
+ static int CHUNKSIZEMAX = 100;
+
+ public CMUserSetHelper(String usersetid, String cmIP) {
+
+ this.usersetid = usersetid;
+ this.cmIP = cmIP;
+ this.apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?name=";
+ this.addusertouserset = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users";
+ this.authUrl = "https://" + cmIP + "/api/v1/auth/tokens";
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ String username = args[0];
+ String password = args[1];
+ String cmip = args[2];
+ String usersetid = args[3];
+ String filePath = args[4];
+ CMUserSetHelper cmusersetHelper = new CMUserSetHelper(usersetid, cmip);
+ int totalrecords = 0;
+
+ String jwthtoken = geAuthToken(cmusersetHelper.authUrl, username, password);
+
+ String newtoken = "Bearer " + removeQuotes(jwthtoken);
+
+ // String filePath =
+ // "C:\\Users\\t0185905\\workspace\\CT-VL-GCP\\src\\main\\java\\com\\example\\emailAddresses.txt";
+ RandomAccessFile file = new RandomAccessFile(filePath, "r");
+ if (cmusersetHelper.chunksize > CHUNKSIZEMAX)
+ cmusersetHelper.chunksize = CHUNKSIZEMAX;
+ int totoalnbrofrecords = numberOfLines(file);
+ if (cmusersetHelper.chunksize > totoalnbrofrecords) {
+ cmusersetHelper.chunksize = totoalnbrofrecords / 2;
+ }
+ //totalrecords = cmusersetHelper.addAUserToUserSet(cmusersetHelper.addusertouserset, newtoken);
+ totalrecords = cmusersetHelper.addAUserToUserSetFromFile(cmusersetHelper.addusertouserset, newtoken, filePath);
+ System.out.println("Totalrecords inserted into Userset " + cmusersetHelper.usersetid + " = " + totalrecords);
+
+ }
+
+ /**
+ * Returns an boolean if user found
+ *
+ *
+ * @param user user to find
+ * @param newtoken jwt token to use
+ * @return boolean true if found in userset
+ * @throws CustomException
+ */
+ public boolean findUserInUserSet(String user, String newtoken) throws CustomException {
+ boolean found = false;
+
+ String apiUrl = this.apiUrlGetUsers + user;
+
+ apiUrl = removeQuotes(apiUrl);
+
+ try {
+ URL url = new URL(apiUrl);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestMethod("GET");
+ connection.setRequestProperty("Authorization", newtoken);
+ connection.setRequestProperty("accept", "application/json");
+
+ connection.setDoInput(true);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+
+ StringBuilder response = new StringBuilder();
+ String line;
+ while ((line = reader.readLine()) != null) {
+ response.append(line);
+ }
+ reader.close();
+
+ Gson gson = new Gson();
+ if (debug)
+ System.out.println("response " + response);
+ JsonObject input = null;
+ JsonElement total = null;
+ JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ input = rootNode.getAsJsonObject();
+ if (input.isJsonObject()) {
+ total = input.get("total");
+ }
+ }
+ JsonPrimitive column = total.getAsJsonPrimitive();
+ String totalstr = column.getAsJsonPrimitive().toString();
+
+ Integer i = Integer.valueOf(totalstr);
+
+ if (i > 0) {
+ found = true;
+
+ }
+ connection.disconnect();
+ } catch (Exception e) {
+ if (e.getMessage().contains("403")) {
+ throw new CustomException("1002, User Not in Application Data Protection Clients ", 1002);
+ } else
+ e.printStackTrace();
+ }
+
+ return found;
+
+ }
+
+ /**
+ * Loads users from a file to the userset. Returns an int of number of users
+ * added.
+ *
+ *
+ * @param url url to userset api
+ * @param newtoken jwt token to use
+ * @param filePath file to load
+ * @return int totalnumberofrecords added to userset
+ */
+
+ public int addAUserToUserSetFromFile(String url, String newtoken, String filePath) throws IOException {
+
+ int totalnbrofrecords = 0;
+ try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
+ String line;
+ int count = 0;
+ StringBuilder payloadBuilder = new StringBuilder();
+
+ payloadBuilder.append("{\"users\": [");
+
+ while ((line = br.readLine()) != null) {
+ totalnbrofrecords++;
+ payloadBuilder.append("\"").append(line).append("\",");
+ count++;
+
+ // If 'n' records have been read, print the payload
+ if (count == chunksize) {
+ makeCMCall(payloadBuilder, newtoken, url);
+ // Reset payload builder and count for the next batch of records
+ payloadBuilder = new StringBuilder("{\"users\": [");
+ count = 0;
+ }
+ }
+
+ // If there are remaining records, print the payload
+ if (count > 0) {
+ payloadBuilder.deleteCharAt(payloadBuilder.length() - 1); // Remove the trailing comma
+ payloadBuilder.append("}");
+ makeCMCall(payloadBuilder, newtoken, url);
+ if (debug)
+ System.out.println(payloadBuilder.toString());
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ return totalnbrofrecords;
+
+ }
+
+ /**
+ * Loads users from a file to the userset. Returns an int of number of users
+ * added.
+ *
+ *
+ * @param payloadBuilder payload for CM call that contains users to add
+ * @param newtoken jwt token to use
+ * @param url url to peform inserts
+ * @return int response code
+ */
+
+ public static int makeCMCall(StringBuilder payloadBuilder, String newtoken, String url) throws IOException {
+
+ payloadBuilder.deleteCharAt(payloadBuilder.length() - 1); // Remove the trailing comma
+ payloadBuilder.append("]}");
+ if (debug)
+ System.out.println(payloadBuilder.toString());
+ String payload = payloadBuilder.toString();
+
+ // Create URL object
+ URL apiUrl = new URL(url);
+ // Create connection object
+ HttpURLConnection connection = (HttpURLConnection) apiUrl.openConnection();
+ connection.setRequestMethod("POST");
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setDoOutput(true);
+ connection.setRequestProperty("Authorization", newtoken);
+ // Send request
+ OutputStream outputStream = connection.getOutputStream();
+ outputStream.write(payload.getBytes());
+ outputStream.flush();
+ outputStream.close();
+
+ // Get response
+ int responseCode = connection.getResponseCode();
+ BufferedReader reader;
+ if (responseCode >= 200 && responseCode < 300) {
+ reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+ } else {
+ reader = new BufferedReader(new InputStreamReader(connection.getErrorStream()));
+ }
+
+ // Read response
+ String lineresponse;
+ StringBuilder response = new StringBuilder();
+ while ((lineresponse = reader.readLine()) != null) {
+ response.append(lineresponse);
+ }
+ reader.close();
+
+ // Print response
+ if (debug) {
+ System.out.println("Response Code: " + responseCode);
+ System.out.println("Response Body: " + response.toString());
+ }
+ // Disconnect connection
+ connection.disconnect();
+ // Reset payload builder and count for the next batch of records
+ payloadBuilder = new StringBuilder("{\"users\": [");
+
+ return responseCode;
+
+ }
+
+ /**
+ * Simple sample of showing how to load a couple of users to the userset.
+ * Returns an int of number of users added.
+ *
+ *
+ * @param url url to userset api
+ * @param newtoken jwt token to use
+ *
+ * @return int response code
+ */
+ public static int addAUserToUserSet(String url, String newtoken) throws IOException {
+ boolean found = false;
+
+ String payload = "{\"users\": [\"akhip@company.com\",\"user2@company.com\"]}";
+
+ // Create URL object
+ URL apiUrl = new URL(url);
+ // Create connection object
+ HttpURLConnection connection = (HttpURLConnection) apiUrl.openConnection();
+ connection.setRequestMethod("POST");
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setDoOutput(true);
+ connection.setRequestProperty("Authorization", newtoken);
+ // Send request
+ OutputStream outputStream = connection.getOutputStream();
+ outputStream.write(payload.getBytes());
+ outputStream.flush();
+ outputStream.close();
+
+ // Get response
+ int responseCode = connection.getResponseCode();
+ BufferedReader reader;
+ if (responseCode >= 200 && responseCode < 300) {
+ reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+ } else {
+ reader = new BufferedReader(new InputStreamReader(connection.getErrorStream()));
+ }
+
+ // Read response
+ String line;
+ StringBuilder response = new StringBuilder();
+ while ((line = reader.readLine()) != null) {
+ response.append(line);
+ }
+ reader.close();
+
+ // Print response
+ if (debug) {
+ System.out.println("Response Code: " + responseCode);
+ System.out.println("Response Body: " + response.toString());
+ }
+ // Disconnect connection
+ connection.disconnect();
+
+ return responseCode;
+
+ }
+
+ public static String removeQuotes(String input) {
+
+ // Remove double quotes from the input string
+ input = input.replace("\"", "");
+
+ return input;
+ }
+
+ private static int numberOfLines(RandomAccessFile file) throws IOException {
+ int numberOfLines = 0;
+ while (file.readLine() != null) {
+ numberOfLines++;
+ }
+ file.seek(0);
+ return numberOfLines;
+ }
+
+ /**
+ * Get JWT from CM
+ *
+ *
+ * @param apiUrl url to CM auth
+ * @param username username on CM
+ * @param pwd password on CM
+ * @return String jwt
+ */
+
+ public static String geAuthToken(String apiUrl, String usernb, String pwd) throws Exception
+
+ {
+
+ String jStr = "{\"username\":\"" + usernb + "\",\"password\":\"" + pwd + "\"}";
+ disableCertValidation();
+
+ String jwtstr = null;
+ try {
+ URL url = new URL(apiUrl);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestProperty("Content-length", String.valueOf(jStr.length()));
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setRequestMethod("GET");
+ connection.setDoOutput(true);
+ connection.setDoInput(true);
+ DataOutputStream output = new DataOutputStream(connection.getOutputStream());
+ output.writeBytes(jStr);
+ output.close();
+ BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+ StringBuilder response = new StringBuilder();
+ String line;
+ while ((line = reader.readLine()) != null) {
+ response.append(line);
+ }
+ reader.close();
+
+ JsonObject input = null;
+ JsonElement jwt = null;
+ JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ input = rootNode.getAsJsonObject();
+ if (input.isJsonObject()) {
+ jwt = input.get("jwt");
+ }
+ }
+ JsonPrimitive column = jwt.getAsJsonPrimitive();
+ jwtstr = column.getAsJsonPrimitive().toString();
+ connection.disconnect();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ return jwtstr;
+
+ }
+
+ /**
+ * Get JWT from CM
+ *
+ *
+ * @param apiUrl url to CM auth
+ * @param username username on CM
+ * @param pwd password on CM
+ * @return String jwt
+ */
+
+ public static String geAuthToken(String apiUrl) throws Exception
+
+ {
+
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+ String jStr = "{\"username\":\"" + userName + "\",\"password\":\"" + password + "\"}";
+
+ disableCertValidation();
+
+ String jwtstr = null;
+ try {
+ URL url = new URL(apiUrl);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestProperty("Content-length", String.valueOf(jStr.length()));
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setRequestMethod("GET");
+ connection.setDoOutput(true);
+ connection.setDoInput(true);
+ DataOutputStream output = new DataOutputStream(connection.getOutputStream());
+ output.writeBytes(jStr);
+ output.close();
+ BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+ StringBuilder response = new StringBuilder();
+ String line;
+ while ((line = reader.readLine()) != null) {
+ response.append(line);
+ }
+ reader.close();
+ if (debug)
+ System.out.println("response " + response);
+ JsonObject input = null;
+ JsonElement jwt = null;
+ JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ input = rootNode.getAsJsonObject();
+ if (input.isJsonObject()) {
+ jwt = input.get("jwt");
+ }
+ }
+ JsonPrimitive column = jwt.getAsJsonPrimitive();
+ jwtstr = column.getAsJsonPrimitive().toString();
+ connection.disconnect();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ return jwtstr;
+
+ }
+
+ public static void disableCertValidation() throws Exception {
+ TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() {
+ public java.security.cert.X509Certificate[] getAcceptedIssuers() {
+ return null;
+ }
+
+ public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) {
+ }
+
+ public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) {
+ }
+ } };
+
+ // Install the all-trusting trust manager
+ try {
+ SSLContext sc = SSLContext.getInstance("SSL");
+ sc.init(null, trustAllCerts, new java.security.SecureRandom());
+ HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ // Create all-trusting host name verifier
+ HostnameVerifier allHostsValid = new HostnameVerifier() {
+ public boolean verify(String hostname, SSLSession session) {
+ return true;
+ }
+ };
+
+ // Install the all-trusting host verifier
+ HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid);
+ }
+
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/CustomException.java b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/CustomException.java
new file mode 100644
index 00000000..6aed3bdd
--- /dev/null
+++ b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/CustomException.java
@@ -0,0 +1,15 @@
+package com.example;
+
+
+class CustomException extends Exception {
+ private int errorCode;
+
+ public CustomException(String message, int errorCode) {
+ super(message);
+ this.errorCode = errorCode;
+ }
+
+ public int getErrorCode() {
+ return errorCode;
+ }
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCADPBulkFPE.java b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCADPBulkFPE.java
new file mode 100644
index 00000000..51a59b9a
--- /dev/null
+++ b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCADPBulkFPE.java
@@ -0,0 +1,493 @@
+package com.example;
+
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+
+import com.google.cloud.functions.HttpFunction;
+import com.google.cloud.functions.HttpRequest;
+import com.google.cloud.functions.HttpResponse;
+import com.ingrian.security.nae.IngrianProvider;
+import com.ingrian.security.nae.AbstractNAECipher;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec;
+import com.ingrian.security.nae.NAEKey;
+import com.ingrian.security.nae.NAESession;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
+import com.ingrian.security.nae.IngrianProvider.Builder;
+import com.ingrian.security.nae.NAECipher;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonPrimitive;
+
+import java.math.BigInteger;
+import java.security.spec.AlgorithmParameterSpec;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Logger;
+/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application
+ * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
+ * data so applications or business intelligence tools do not have to change in order to use these columns.
+*
+* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
+* for all possible data sizes and combinations of encryption algorithms and IV, etc.
+* Was tested with CM 2.14 & CADP 8.16.001
+* For more information on CADP see link below.
+https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html
+* For more information on Snowflake External Functions see link below.
+https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp
+
+Notes: This example uses the CADP bulk API.
+Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data
+element has Unicode characters then the supported size limit would be 1750 characters
+
+Size of spec array should be same as the number of elements if user wants to use separate spec values
+for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data
+index.
+ *
+ *@author mwarner
+ *
+ */
+public class ThalesGCPSnowCADPBulkFPE implements HttpFunction {
+// @Override
+ private static byte[][] data;
+ private static AlgorithmParameterSpec[] spec;
+ private static Integer[] snowrownbrs;
+
+ private static int BATCHLIMIT = 10000;
+ private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPBulkFPE.class.getName());
+ private static final Gson gson = new Gson();
+ private static final String BADDATATAG = new String ("9999999999999999");
+
+ public void service(HttpRequest request, HttpResponse response) throws Exception {
+
+ String tweakAlgo = null;
+ String tweakData = null;
+ String snowflakereturnstring = null;
+ JsonArray snowflakedata = null;
+ NAESession session = null;
+
+ Map encryptedErrorMapTotal = new HashMap();
+ JsonObject body = null;
+ int statusCode = 200;
+ int numberofchunks = 0;
+
+ String keyName = "testfaas";
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be
+ // returned when the user above does not have access to the key and if doing a lookup in the userset
+ // and the user does not exist. If returnciphertextforuserwithnokeyaccess = no then an error will be
+ // returned to the query, else the results set will provide ciphertext.
+ // yes/no
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB?
+ // yes/no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it
+ // is the userset in CM but could be a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ String datatype = System.getenv("datatype");
+ //mode of operation valid values are : encrypt or decrypt
+ String mode = System.getenv("mode");
+
+ int cipherType = 0;
+ if (mode.equals("encrypt"))
+ cipherType = Cipher.ENCRYPT_MODE;
+ else
+ cipherType = Cipher.DECRYPT_MODE;
+
+
+ int batchsize = Integer.parseInt(System.getenv("BATCHSIZE"));
+
+ try {
+
+
+
+ // This code is only to be used when input data contains user info.
+ /*
+ * if (usersetlookupbool) { // make sure cmuser is in Application Data
+ * Protection Clients Group
+ *
+ * boolean founduserinuserset = findUserInUserSet(bigquerysessionUser, userName,
+ * password, usersetID, userSetLookupIP); // System.out.println("Found User " +
+ * founduserinuserset); if (!founduserinuserset) throw new
+ * CustomException("1001, User Not in User Set", 1001);
+ *
+ * } else { usersetlookupbool = false; }
+ */
+ //How many records in a chunk. Testing has indicated point of diminishing returns at 100 or 200, but
+ //may vary depending on size of data.
+
+ try {
+ JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class);
+ JsonObject requestJson = null;
+
+ if (requestParsed != null && requestParsed.isJsonObject()) {
+ requestJson = requestParsed.getAsJsonObject();
+ }
+
+ if (requestJson != null && requestJson.has("data")) {
+ snowflakedata = requestJson.getAsJsonArray("data");
+
+ }
+
+ } catch (JsonParseException e) {
+ logger.severe("Error parsing JSON: " + e.getMessage());
+ }
+
+ // System.setProperty("com.ingrian.security.nae.IngrianNAE_Properties_Conf_Filename",
+ // "IngrianNAE.properties");
+ System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
+ "CADP_for_JAVA.properties");
+ IngrianProvider builder = new Builder().addConfigFileInputStream(
+ getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
+ // IngrianProvider builder = new
+ // Builder().addConfigFileInputStream(getClass().getClassLoader().getResourceAsStream("IngrianNAE.properties")).build();
+ session = NAESession.getSession(userName, password.toCharArray());
+ NAEKey key = NAEKey.getSecretKey(keyName, session);
+ int row_number = 0;
+
+ StringBuffer snowflakereturndata = new StringBuffer();
+ int numberOfLines = snowflakedata.size();
+ int totalRowsLeft = numberOfLines;
+
+ if (batchsize > numberOfLines)
+ batchsize = numberOfLines;
+ if (batchsize >= BATCHLIMIT)
+ batchsize = BATCHLIMIT;
+ spec = new FPEParameterAndFormatSpec[batchsize];
+ data = new byte[batchsize][];
+ snowrownbrs = new Integer[batchsize];
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+ JsonArray innerDataArray = new JsonArray();
+
+ String algorithm = null;
+
+ if (datatype.equals("char"))
+ algorithm = "FPE/FF1/CARD62";
+ else if (datatype.equals("charint"))
+ algorithm = "FPE/FF1/CARD10";
+ else
+ algorithm = "FPE/FF1/CARD10";
+
+
+
+ AbstractNAECipher thalesCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider");
+ int i = 0;
+
+ if (batchsize > numberOfLines)
+ batchsize = numberOfLines;
+ if (batchsize >= BATCHLIMIT)
+ batchsize = BATCHLIMIT;
+
+ FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo).build();
+
+ spec = new FPEParameterAndFormatSpec[batchsize];
+ data = new byte[batchsize][];
+ snowrownbrs = new Integer[batchsize];
+ String sensitive = null;
+ thalesCipher.init(cipherType, key, spec[0]);
+
+ //int i = 0;
+ int count = 0;
+ boolean newchunk = true;
+ int dataIndex = 0;
+ int specIndex = 0;
+ int snowRowIndex = 0;
+ //String sensitive = null;
+
+
+ while (i < numberOfLines) {
+ int index = 0;
+
+ if (newchunk) {
+
+ if (totalRowsLeft < batchsize) {
+ spec = new FPEParameterAndFormatSpec[totalRowsLeft];
+ data = new byte[totalRowsLeft][];
+ snowrownbrs = new Integer[totalRowsLeft];
+ } else {
+ spec = new FPEParameterAndFormatSpec[batchsize];
+ data = new byte[batchsize][];
+ snowrownbrs = new Integer[batchsize];
+ }
+ newchunk = false;
+ }
+
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ for (int j = 0; j < snowflakerow.size(); j++) {
+
+ if (j == 1) {
+
+ sensitive = checkValid(snowflakerow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ sensitive = BADDATATAG+sensitive;
+ data[dataIndex++] = sensitive.getBytes();
+ // Can not return number since a leading 0 will not work.
+ // innerDataArray.add(new BigInteger(sensitive));
+ } else
+ data[dataIndex++] = BADDATATAG.getBytes();
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+ data[dataIndex++] = sensitive.getBytes();
+ } else {
+ data[dataIndex++] = sensitive.getBytes();
+ }
+ spec[specIndex++] = param;
+
+ } else {
+ data[dataIndex++] = sensitive.getBytes();
+ spec[specIndex++] = param;
+ }
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ row_number = snowflakecolumn.getAsInt();
+ snowrownbrs[snowRowIndex++] = row_number;
+
+ }
+
+ }
+
+ if (count == batchsize - 1) {
+
+ // Map to store exceptions while encryption
+ Map encryptedErrorMap = new HashMap();
+
+ byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap);
+
+ for (Map.Entry entry : encryptedErrorMap.entrySet()) {
+ Integer mkey = entry.getKey();
+ String mvalue = entry.getValue();
+ encryptedErrorMapTotal.put(mkey, mvalue);
+ }
+
+ for (int enc = 0; enc < encryptedData.length; enc++) {
+
+ innerDataArray.add(snowrownbrs[enc]);
+ innerDataArray.add(new String(encryptedData[enc]));
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ index++;
+
+ }
+
+ numberofchunks++;
+ newchunk = true;
+ count = 0;
+ dataIndex = 0;
+ specIndex = 0;
+ snowRowIndex = 0;
+ } else
+ count++;
+
+ totalRowsLeft--;
+ i++;
+ }
+ if (count > 0) {
+ numberofchunks++;
+ int index = 0;
+ Map encryptedErrorMap = new HashMap();
+ byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap);
+ for (int enc = 0; enc < encryptedData.length; enc++) {
+ innerDataArray.add(snowrownbrs[enc]);
+ innerDataArray.add(new String(encryptedData[enc]));
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ index++;
+
+ }
+ }
+
+
+ bodyObject.add("data", dataArray);
+ String bodyString = bodyObject.toString();
+
+ snowflakereturnstring = bodyString.toString();
+
+
+ } catch (
+
+ Exception e) {
+ System.out.println("in exception with " + e.getMessage());
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401") || (e.getMessage().contains("1001") || (e.getMessage().contains("1002"))) ) {
+
+ try {
+ snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, true, null,datatype);
+ } catch (IllegalBlockSizeException e1) {
+ // TODO Auto-generated catch block
+ e1.printStackTrace();
+ } catch (BadPaddingException e1) {
+ // TODO Auto-generated catch block
+ e1.printStackTrace();
+ }
+
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } finally {
+ if (session != null) {
+ session.closeSession();
+ }
+ }
+ System.out.println("number of chunks = " + numberofchunks);
+ //System.out.println("snowflakereturnstring = " + snowflakereturnstring);
+ response.getWriter().write(snowflakereturnstring);
+
+
+
+ }
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer snowflakereturndatasb = new StringBuffer();
+
+ snowflakereturndatasb.append("{ \"statusCode\":");
+ snowflakereturndatasb.append(statusCode);
+ snowflakereturndatasb.append(",");
+ snowflakereturndatasb.append(" \"body\": {");
+ snowflakereturndatasb.append(" \"data\": [");
+ snowflakereturndatasb.append("] }}");
+ System.out.println("in exception with ");
+ return snowflakereturndatasb.toString();
+ }
+
+ public String checkValid(JsonArray snowrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (snowrow != null && snowrow.size() > 0) {
+ JsonElement element = snowrow.get(1);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+
+
+ public String formatReturnValue(int statusCode, JsonArray snowflakedata, boolean error, Cipher thalesCipher,
+ String datatype) throws IllegalBlockSizeException, BadPaddingException {
+ int row_number = 0;
+
+ String encdata = null;
+ String sensitive = null;
+
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+ JsonArray innerDataArray = new JsonArray();
+
+ for (int i = 0; i < snowflakedata.size(); i++) {
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+ for (int j = 0; j < snowflakerow.size(); j++) {
+ if (j == 1) {
+ sensitive = checkValid(snowflakerow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ innerDataArray.add(sensitive);
+ // Can not return number since a leading 0 will not work.
+ // innerDataArray.add(new BigInteger(sensitive));
+ } else
+ innerDataArray.add(BADDATATAG);
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+ innerDataArray.add("");
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ innerDataArray.add(sensitive);
+ } else {
+ innerDataArray.add(sensitive);
+ }
+
+ } else {
+ if (!error) {
+ byte[] outbuf = thalesCipher.doFinal(sensitive.getBytes());
+ encdata = new String(outbuf);
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ innerDataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ innerDataArray.add(encdata);
+ }
+ } else {
+ encdata = sensitive;
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ innerDataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ innerDataArray.add(encdata);
+ }
+ }
+ }
+
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ row_number = snowflakecolumn.getAsInt();
+ innerDataArray.add(row_number);
+ }
+ }
+ }
+
+ bodyObject.add("data", dataArray);
+ String bodyString = bodyObject.toString();
+
+ return bodyString;
+
+ }
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+
+ boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
+
+ return founduserinuserset;
+
+ }
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCADPFPE.java b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCADPFPE.java
new file mode 100644
index 00000000..aaa18fb2
--- /dev/null
+++ b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCADPFPE.java
@@ -0,0 +1,308 @@
+package com.example;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+
+import com.google.cloud.functions.HttpFunction;
+import com.google.cloud.functions.HttpRequest;
+import com.google.cloud.functions.HttpResponse;
+import com.ingrian.security.nae.IngrianProvider;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec;
+import com.ingrian.security.nae.NAEKey;
+import com.ingrian.security.nae.NAESession;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
+import com.ingrian.security.nae.IngrianProvider.Builder;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonPrimitive;
+
+import java.math.BigInteger;
+import java.util.logging.Logger;
+
+/* This test app to test the logic for a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application
+ * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
+ * data so applications or business intelligence tools do not have to change in order to use these columns.
+*
+* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
+* for all possible data sizes and combinations of encryption algorithms and IV, etc.
+* Was tested with CM 2.14 & CADP 8.16
+* For more information on CADP see link below.
+https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html
+* For more information on Snowflake External Functions see link below.
+https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp
+ *
+ *@author mwarner
+ *
+ */
+public class ThalesGCPSnowCADPFPE implements HttpFunction {
+// @Override
+
+ private static final Gson gson = new Gson();
+ private static final String BADDATATAG = new String ("9999999999999999");
+
+
+ public void service(HttpRequest request, HttpResponse response) throws Exception {
+
+ String encdata = "";
+ String tweakAlgo = null;
+ String tweakData = null;
+ String snowflakereturnstring = null;
+ JsonArray snowflakedata = null;
+ int statusCode = 200;
+
+ NAESession session = null;
+ String keyName = "testfaas";
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned
+ // when the user above does not have access to the key and if doing a
+ // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no
+ // then an error will be returned to the query, else the results set will provide ciphertext.
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ // yes,no
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB
+ // yes,no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be
+ // a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ String datatype = System.getenv("datatype");
+ //mode of operation valid values are : encrypt or decrypt
+ String mode = System.getenv("mode");
+
+ int cipherType = 0;
+ if (mode.equals("encrypt"))
+ cipherType = Cipher.ENCRYPT_MODE;
+ else
+ cipherType = Cipher.DECRYPT_MODE;
+
+ try {
+
+
+ //This code is only to be used when input data contains user info.
+ /*
+ * if (usersetlookupbool) { // make sure cmuser is in Application Data
+ * Protection Clients Group
+ *
+ * boolean founduserinuserset = findUserInUserSet(bigquerysessionUser, userName,
+ * password, usersetID, userSetLookupIP); // System.out.println("Found User " +
+ * founduserinuserset); if (!founduserinuserset) throw new
+ * CustomException("1001, User Not in User Set", 1001);
+ *
+ * } else { usersetlookupbool = false; }
+ */
+
+
+
+ try {
+ JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class);
+ JsonObject requestJson = null;
+
+ if (requestParsed != null && requestParsed.isJsonObject()) {
+ requestJson = requestParsed.getAsJsonObject();
+ }
+
+ if (requestJson != null && requestJson.has("data")) {
+ snowflakedata = requestJson.getAsJsonArray("data");
+
+ }
+
+ } catch (JsonParseException e) {
+ System.out.println("Error parsing JSON: " + e.getMessage());
+ }
+
+
+ System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
+ "CADP_for_JAVA.properties");
+ IngrianProvider builder = new Builder().addConfigFileInputStream(
+ getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")).build();
+
+ session = NAESession.getSession(userName, password.toCharArray());
+ NAEKey key = NAEKey.getSecretKey(keyName, session);
+ int row_number = 0;
+
+ String algorithm = null;
+
+ if (datatype.equals("char"))
+ algorithm = "FPE/FF1/CARD62";
+ else if (datatype.equals("charint"))
+ algorithm = "FPE/FF1/CARD10";
+ else
+ algorithm = "FPE/FF1/CARD10";
+
+ FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
+ .build();
+
+ Cipher thalesCipher = Cipher.getInstance(algorithm, "IngrianProvider");
+ // initialize cipher to encrypt.
+ thalesCipher.init(cipherType, key, param);
+
+ snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, false, thalesCipher, datatype);
+
+ } catch (Exception e) {
+
+ System.out.println("in exception with " + e.getMessage());
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
+
+ snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, true, null, datatype);
+
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+
+ } finally {
+ if (session != null) {
+ session.closeSession();
+ }
+ }
+ // System.out.println(snowflakereturnstring);
+ response.getWriter().write(snowflakereturnstring);
+
+ }
+
+ public String checkValid(JsonArray snowrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (snowrow != null && snowrow.size() > 0) {
+ JsonElement element = snowrow.get(1);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer snowflakereturndatasb = new StringBuffer();
+
+ snowflakereturndatasb.append("{ \"statusCode\":");
+ snowflakereturndatasb.append(statusCode);
+ snowflakereturndatasb.append(",");
+ snowflakereturndatasb.append(" \"body\": {");
+ snowflakereturndatasb.append(" \"data\": [");
+ snowflakereturndatasb.append("] }}");
+ System.out.println("in exception with ");
+ return snowflakereturndatasb.toString();
+ }
+
+ public String formatReturnValue(int statusCode, JsonArray snowflakedata, boolean error, Cipher thalesCipher,
+ String datatype) throws IllegalBlockSizeException, BadPaddingException {
+ int row_number = 0;
+
+ String encdata = null;
+ String sensitive = null;
+
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+ JsonArray innerDataArray = new JsonArray();
+
+ for (int i = 0; i < snowflakedata.size(); i++) {
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+ for (int j = 0; j < snowflakerow.size(); j++) {
+ if (j == 1) {
+ sensitive = checkValid(snowflakerow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ innerDataArray.add(sensitive);
+ // Can not return number since a leading 0 will not work.
+ // innerDataArray.add(new BigInteger(sensitive));
+ } else
+ innerDataArray.add(BADDATATAG);
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+ innerDataArray.add("");
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ innerDataArray.add(sensitive);
+ } else {
+ innerDataArray.add(sensitive);
+ }
+
+ } else {
+ if (!error) {
+ byte[] outbuf = thalesCipher.doFinal(sensitive.getBytes());
+ encdata = new String(outbuf);
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ innerDataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ innerDataArray.add(encdata);
+ }
+ } else {
+ encdata = sensitive;
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ innerDataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ innerDataArray.add(encdata);
+ }
+ }
+ }
+
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ row_number = snowflakecolumn.getAsInt();
+ innerDataArray.add(row_number);
+ }
+ }
+ }
+
+ bodyObject.add("data", dataArray);
+ String bodyString = bodyObject.toString();
+
+ return bodyString;
+
+ }
+
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+
+ boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
+
+ return founduserinuserset;
+
+ }
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCRDPBulkFPE.java b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCRDPBulkFPE.java
new file mode 100644
index 00000000..e76791db
--- /dev/null
+++ b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCRDPBulkFPE.java
@@ -0,0 +1,539 @@
+package com.example;
+
+import com.google.cloud.functions.HttpFunction;
+import com.google.cloud.functions.HttpRequest;
+import com.google.cloud.functions.HttpResponse;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonPrimitive;
+
+import okhttp3.MediaType;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/* This is a Thales CRDP UDF for Snowflake. It uses the Thales CRDP Bulk API.
+ * It is an example of how to use Thales CipherTrust REST Application Dataprotection (CRDP)
+ * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
+ * data so applications or business intelligence tools do not have to change in order to use these columns.
+*
+* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
+* for all possible data sizes and combinations of encryption algorithms and IV, etc.
+* Was tested with CM 2.14 and CRDP 1.0
+* For more information on CRDP see link below.
+https://thalesdocs.com/ctp/con/crdp/latest/index.html
+* For more information on Snowflake External Functions see link below.
+https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp
+ *
+ *@author mwarner
+ *
+ */
+public class ThalesGCPSnowCRDPBulkFPE implements HttpFunction {
+// @Override
+
+ private static final Gson gson = new Gson();
+ private static final String BADDATATAG = new String("9999999999999999");
+ private static int BATCHLIMIT = 10000;
+ private static final String REVEALRETURNTAG = new String("data");
+ private static final String PROTECTRETURNTAG = new String("protected_data");
+
+ public void service(HttpRequest request, HttpResponse response) throws Exception {
+ Map snowErrorMap = new HashMap();
+ String encdata = "";
+ int error_count = 0;
+ int statusCode = 200;
+
+ JsonObject result = new JsonObject();
+ JsonArray replies = new JsonArray();
+ JsonArray rownbranddata = new JsonArray();
+
+ String keyName = "testfaas";
+ String crdpip = System.getenv("CRDPIP");
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned
+ // when the user above does not have access to the key and if doing a
+ // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no
+ // then an error will be returned to the query, else the results set will provide ciphertext.
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ // yes,no
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB
+ // yes,no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be
+ // a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ String keymetadatalocation = System.getenv("keymetadatalocation");
+ String external_version_from_ext_source = System.getenv("keymetadata");
+ String protection_profile = System.getenv("protection_profile");
+ String mode = System.getenv("mode");
+ String datatype = System.getenv("datatype");
+
+ String inputDataKey = null;
+ String outputDataKey = null;
+ String protectedData = null;
+ String externalkeymetadata = null;
+ String jsonBody = null;
+ String jsonTagForProtectReveal = null;
+
+ boolean bad_data = false;
+ String showrevealkey = "yes";
+
+ if (mode.equals("protectbulk")) {
+ inputDataKey = "data_array";
+ outputDataKey = "protected_data_array";
+ jsonTagForProtectReveal = PROTECTRETURNTAG;
+ if (keymetadatalocation.equalsIgnoreCase("internal")) {
+ showrevealkey = System.getenv("showrevealinternalkey");
+ if (showrevealkey == null)
+ showrevealkey = "yes";
+ }
+ } else {
+ inputDataKey = "protected_data_array";
+ outputDataKey = "data_array";
+ jsonTagForProtectReveal = REVEALRETURNTAG;
+ }
+
+ boolean showrevealkeybool = showrevealkey.equalsIgnoreCase("yes");
+
+ String snowflakeuser = "snowflakeuser";
+ JsonArray snowflakedata = null;
+ String snowflakereturnstring = null;
+ Response crdp_response = null;
+ try {
+
+ // This code is only to be used when input data contains user info.
+ /*
+ * if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group
+ *
+ * boolean founduserinuserset = findUserInUserSet(bigquerysessionUser, userName, password, usersetID,
+ * userSetLookupIP); // System.out.println("Found User " + founduserinuserset); if (!founduserinuserset)
+ * throw new CustomException("1001, User Not in User Set", 1001);
+ *
+ * } else { usersetlookupbool = false; }
+ */
+
+ try {
+ JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class);
+ JsonObject requestJson = null;
+
+ if (requestParsed != null && requestParsed.isJsonObject()) {
+ requestJson = requestParsed.getAsJsonObject();
+ }
+
+ if (requestJson != null && requestJson.has("data")) {
+ snowflakedata = requestJson.getAsJsonArray("data");
+
+ }
+
+ } catch (JsonParseException e) {
+ System.out.println("Error parsing JSON: " + e.getMessage());
+ // logger.severe("Error parsing JSON: " + e.getMessage());
+ }
+
+ // Serialization
+
+ int numberofchunks = 0;
+ StringBuffer protection_policy_buff = new StringBuffer();
+ String notvalid = "notvalid";
+ // StringBuffer snowflakereturndata = new StringBuffer();
+
+ int numberOfLines = snowflakedata.size();
+ int totalRowsLeft = numberOfLines;
+ int batchsize = Integer.parseInt(System.getenv("BATCHSIZE"));
+
+ if (batchsize > numberOfLines)
+ batchsize = numberOfLines;
+ if (batchsize >= BATCHLIMIT)
+ batchsize = BATCHLIMIT;
+ // Serialization
+
+ String crdpjsonBody = null;
+ // String external_version_from_ext_source = "1004001";
+ // String external_version_from_ext_source = "1001001";
+
+ int i = 0;
+ int count = 0;
+ int totalcount = 0;
+
+ int dataIndex = 0; // assumes index from snowflake will always be sequential.
+ JsonObject crdp_payload = new JsonObject();
+ String sensitive = null;
+ JsonArray crdp_payload_array = new JsonArray();
+
+ OkHttpClient client = new OkHttpClient().newBuilder().build();
+ MediaType mediaType = MediaType.parse("application/json");
+ String urlStr = "http://" + crdpip + ":8090/v1/" + mode;
+
+ while (i < numberOfLines) {
+
+ for (int b = 0; b < batchsize && b < totalRowsLeft; b++) {
+
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ sensitive = checkValid(snowflakerow);
+ protection_profile = protection_profile.trim();
+ // Format the output
+ String formattedElement = String.format("\"protection_policy_name\" : \"%s\"", protection_profile);
+ protection_policy_buff.append(formattedElement);
+ protection_policy_buff.append(",");
+
+ if (mode.equals("protectbulk")) {
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ // System.out.println("adding null not charint or nbr");
+ sensitive = sensitive.replace("notvalid", "");
+ sensitive = BADDATATAG + sensitive;
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ // sensitive = sensitive.replace("notvalid", "");
+
+ }
+ encdata = sensitive;
+
+ }
+ crdp_payload_array.add(sensitive);
+ } else {
+ JsonObject protectedDataObject = new JsonObject();
+ protectedDataObject.addProperty(PROTECTRETURNTAG, sensitive);
+ if (keymetadatalocation.equalsIgnoreCase("external")) {
+ protectedDataObject.addProperty("external_version", external_version_from_ext_source);
+ }
+ crdp_payload_array.add(protectedDataObject);
+
+ }
+
+ if (count == batchsize - 1) {
+ crdp_payload.add(inputDataKey, crdp_payload_array);
+ String inputdataarray = null;
+ if (mode.equals("revealbulk")) {
+ crdp_payload.addProperty("username", snowflakeuser);
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ jsonBody = protection_policy_buff.toString();
+ jsonBody = jsonBody.replaceFirst("\\{", " ");
+
+ } else {
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ inputdataarray = protection_policy_buff.toString();
+ jsonBody = inputdataarray.replace("{", " ");
+ }
+ jsonBody = "{" + jsonBody;
+
+ RequestBody body = RequestBody.create(mediaType, jsonBody);
+
+ Request crdp_request = new Request.Builder().url(urlStr).method("POST", body)
+ .addHeader("Content-Type", "application/json").build();
+ crdp_response = client.newCall(crdp_request).execute();
+ String crdpreturnstr = null;
+ if (crdp_response.isSuccessful()) {
+ // Parse JSON response
+ String responseBody = crdp_response.body().string();
+ JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class);
+ JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey);
+
+ String status = jsonObject.get("status").getAsString();
+ int success_count = jsonObject.get("success_count").getAsInt();
+ error_count = jsonObject.get("error_count").getAsInt();
+ if (error_count > 0)
+ System.out.println("errors " + error_count);
+
+ for (JsonElement element : protectedDataArray) {
+
+ JsonObject protectedDataObject = element.getAsJsonObject();
+ if (protectedDataObject.has(jsonTagForProtectReveal)) {
+
+ protectedData = protectedDataObject.get(jsonTagForProtectReveal).getAsString();
+ // System.out.println(protectedData);
+
+ if (keymetadatalocation.equalsIgnoreCase("internal")
+ && mode.equalsIgnoreCase("protectbulk") && !showrevealkeybool) {
+ if (protectedData.length() > 7)
+ protectedData = protectedData.substring(7);
+ }
+
+ rownbranddata.add(dataIndex);
+ rownbranddata.add(new String(protectedData));
+ replies.add(rownbranddata);
+ rownbranddata = new JsonArray();
+ if (mode.equals("protectbulk")) {
+ if (keymetadatalocation.equalsIgnoreCase("external")
+ && mode.equalsIgnoreCase("protectbulk")) {
+ externalkeymetadata = protectedDataObject.get("external_version")
+ .getAsString();
+ // System.out.println("Protected Data ext key metadata need to store this: "
+ // + externalkeymetadata);
+
+ }
+ }
+ } else if (protectedDataObject.has("error_message")) {
+ String errorMessage = protectedDataObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ snowErrorMap.put(i, errorMessage);
+ bad_data = true;
+ } else
+ System.out.println("unexpected json value from results: ");
+ dataIndex++;
+
+ }
+
+ crdp_payload_array = new JsonArray();
+ protection_policy_buff = new StringBuffer();
+ numberofchunks++;
+ totalcount = totalcount + count;
+ count = 0;
+ } else {// throw error....
+ System.err.println("Request failed with status code: " + crdp_response.code());
+ throw new CustomException("1010, Unexpected Error ", 1010);
+ }
+
+ } else {
+ count++;
+ }
+ totalRowsLeft--;
+ i++;
+ }
+ }
+ if (count > 0) {
+ crdp_payload.add(inputDataKey, crdp_payload_array);
+ String inputdataarray = null;
+ if (mode.equals("revealbulk")) {
+ crdp_payload.addProperty("username", snowflakeuser);
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ jsonBody = protection_policy_buff.toString();
+ jsonBody = jsonBody.replaceFirst("\\{", " ");
+
+ } else {
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ inputdataarray = protection_policy_buff.toString();
+ jsonBody = inputdataarray.replace("{", " ");
+ }
+ jsonBody = "{" + jsonBody;
+
+ RequestBody body = RequestBody.create(mediaType, jsonBody);
+
+ Request crdp_request = new Request.Builder().url(urlStr).method("POST", body)
+ .addHeader("Content-Type", "application/json").build();
+ crdp_response = client.newCall(crdp_request).execute();
+ String crdpreturnstr = null;
+ if (crdp_response.isSuccessful()) {
+ // Parse JSON response
+ String responseBody = crdp_response.body().string();
+ JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class);
+ JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey);
+
+ String status = jsonObject.get("status").getAsString();
+ int success_count = jsonObject.get("success_count").getAsInt();
+ error_count = jsonObject.get("error_count").getAsInt();
+ if (error_count > 0)
+ System.out.println("errors " + error_count);
+
+ for (JsonElement element : protectedDataArray) {
+
+ JsonObject protectedDataObject = element.getAsJsonObject();
+ if (protectedDataObject.has(jsonTagForProtectReveal)) {
+
+ protectedData = protectedDataObject.get(jsonTagForProtectReveal).getAsString();
+ // System.out.println(protectedData);
+
+ rownbranddata.add(dataIndex);
+
+ if (keymetadatalocation.equalsIgnoreCase("internal") && mode.equalsIgnoreCase("protectbulk")
+ && !showrevealkeybool) {
+ if (protectedData.length() > 7)
+ protectedData = protectedData.substring(7);
+ }
+
+ rownbranddata.add(new String(protectedData));
+ replies.add(rownbranddata);
+ rownbranddata = new JsonArray();
+ if (mode.equals("protectbulk")) {
+ if (keymetadatalocation.equalsIgnoreCase("external")
+ && mode.equalsIgnoreCase("protectbulk")) {
+ externalkeymetadata = protectedDataObject.get("external_version").getAsString();
+ // System.out.println("Protected Data ext key metadata need to store this: "
+ // + externalkeymetadata);
+
+ }
+ }
+ } else if (protectedDataObject.has("error_message")) {
+ String errorMessage = protectedDataObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ snowErrorMap.put(i, errorMessage);
+ bad_data = true;
+ } else
+ System.out.println("unexpected json value from results: ");
+ dataIndex++;
+
+ }
+
+ crdp_payload_array = new JsonArray();
+ protection_policy_buff = new StringBuffer();
+ numberofchunks++;
+ totalcount = totalcount + count;
+ count = 0;
+
+ } else {
+ System.err.println("Request failed with status code: " + crdp_response.code());
+ }
+ }
+ crdp_response.close();
+ System.out.println("total chuncks " + numberofchunks);
+
+ result.add("data", replies);
+
+ snowflakereturnstring = result.toString();
+
+ } catch (Exception e) {
+ System.out.println("in exception with " + e.getMessage());
+ snowflakereturnstring = "exception ";
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
+
+ result = new JsonObject();
+ replies = new JsonArray();
+ rownbranddata = new JsonArray();
+
+ for (int i = 0; i < snowflakedata.size(); i++) {
+
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ for (int j = 0; j < snowflakerow.size(); j++) {
+ if (j == 1) {
+ // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
+ // FPE example
+ String sensitive = checkValid(snowflakerow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null")
+ || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+
+ }
+ encdata = sensitive;
+
+ } else {
+ // System.out.println("normal number data" + sensitive);
+ }
+ rownbranddata.add(sensitive);
+ replies.add(rownbranddata);
+ rownbranddata = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ int row_number = snowflakecolumn.getAsInt();
+ rownbranddata.add(row_number);
+ }
+ }
+ }
+
+ result.add("data", replies);
+ JsonObject inputJsonObject = new JsonObject();
+ String bodyString = result.toString();
+ inputJsonObject.addProperty("statusCode", 200);
+ inputJsonObject.addProperty("body", bodyString);
+
+ snowflakereturnstring = inputJsonObject.toString();
+
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+
+ } finally {
+
+ }
+
+ // System.out.println("return string " + snowflakereturnstring);
+ response.getWriter().write(snowflakereturnstring);
+
+ }
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer snowflakereturndatasb = new StringBuffer();
+
+ snowflakereturndatasb.append("{ \"statusCode\":");
+ snowflakereturndatasb.append(statusCode);
+ snowflakereturndatasb.append(",");
+ snowflakereturndatasb.append(" \"body\": {");
+ snowflakereturndatasb.append(" \"data\": [");
+ snowflakereturndatasb.append("] }}");
+ System.out.println("in exception with ");
+ return snowflakereturndatasb.toString();
+ }
+
+ public String checkValid(JsonArray snowrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (snowrow != null && snowrow.size() > 0) {
+ JsonElement element = snowrow.get(1);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+
+ boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
+
+ return founduserinuserset;
+
+ }
+
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCRDPFPE.java b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCRDPFPE.java
new file mode 100644
index 00000000..df265b3a
--- /dev/null
+++ b/database/snowflake/Thales-Snow-GCP-UDF/src/main/java/com/example/ThalesGCPSnowCRDPFPE.java
@@ -0,0 +1,388 @@
+package com.example;
+
+import com.google.cloud.functions.HttpFunction;
+import com.google.cloud.functions.HttpRequest;
+import com.google.cloud.functions.HttpResponse;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonPrimitive;
+
+import okhttp3.MediaType;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/* This is a Thales CRDP UDF for Snowflake.
+ * It is an example of how to use Thales CipherTrust REST Application Dataprotection (CRDP)
+ * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
+ * data so applications or business intelligence tools do not have to change in order to use these columns.
+*
+* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
+* for all possible data sizes and combinations of encryption algorithms and IV, etc.
+* Was tested with CM 2.14 and CRDP 1.0
+* For more information on CRDP see link below.
+https://thalesdocs.com/ctp/con/crdp/latest/index.html
+* For more information on Snowflake External Functions see link below.
+https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp
+ *
+ *@author mwarner
+ *
+ */
+public class ThalesGCPSnowCRDPFPE implements HttpFunction {
+// @Override
+
+ private static final Gson gson = new Gson();
+ private static final String BADDATATAG = new String("9999999999999999");
+ private static final String REVEALRETURNTAG = new String("data");
+ private static final String PROTECTRETURNTAG = new String("protected_data");
+
+ public void service(HttpRequest request, HttpResponse response) throws Exception {
+ Map snowErrorMap = new HashMap();
+ String encdata = "";
+ String snowflakereturnstring = null;
+ int statusCode = 200;
+ JsonObject result = new JsonObject();
+ JsonArray replies = new JsonArray();
+ JsonArray rownbranddata = new JsonArray();
+
+ String keyName = "testfaas";
+ String crdpip = System.getenv("CRDPIP");
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned
+ // when the user above does not have access to the key and if doing a
+ // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no
+ // then an error will be returned to the query, else the results set will provide ciphertext.
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ // yes,no
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB
+ // yes,no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be
+ // a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ String keymetadatalocation = System.getenv("keymetadatalocation");
+ String external_version_from_ext_source = System.getenv("keymetadata");
+ String protection_profile = System.getenv("protection_profile");
+ String mode = System.getenv("mode");
+ String datatype = System.getenv("datatype");
+
+ String dataKey = null;
+
+ boolean bad_data = false;
+ String jsonTagForProtectReveal = null;
+
+ String showrevealkey = "yes";
+
+ if (mode.equals("protect")) {
+ dataKey = "data";
+ jsonTagForProtectReveal = PROTECTRETURNTAG;
+ if (keymetadatalocation.equalsIgnoreCase("internal")) {
+ showrevealkey = System.getenv("showrevealinternalkey");
+ if (showrevealkey == null)
+ showrevealkey = "yes";
+ }
+ } else {
+ dataKey = "protected_data";
+ jsonTagForProtectReveal = REVEALRETURNTAG;
+ }
+ boolean showrevealkeybool = showrevealkey.equalsIgnoreCase("yes");
+
+ String snowflakeuser = "snowflakeuser";
+ JsonArray snowflakedata = null;
+
+ try {
+
+ // This code is only to be used when input data contains user info.
+ /*
+ * if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group
+ *
+ * boolean founduserinuserset = findUserInUserSet(bigquerysessionUser, userName, password, usersetID,
+ * userSetLookupIP); // System.out.println("Found User " + founduserinuserset); if (!founduserinuserset)
+ * throw new CustomException("1001, User Not in User Set", 1001);
+ *
+ * } else { usersetlookupbool = false; }
+ */
+
+ try {
+ JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class);
+ JsonObject requestJson = null;
+
+ if (requestParsed != null && requestParsed.isJsonObject()) {
+ requestJson = requestParsed.getAsJsonObject();
+ }
+
+ if (requestJson != null && requestJson.has("data")) {
+ snowflakedata = requestJson.getAsJsonArray("data");
+
+ }
+
+ } catch (JsonParseException e) {
+ System.out.println("Error parsing JSON: " + e.getMessage());
+
+ }
+
+ int row_number = 0;
+
+ String protectedData = null;
+ String externalkeymetadata = null;
+ String crdpjsonBody = null;
+ // String external_version_from_ext_source = "1004001";
+ // String external_version_from_ext_source = "1001001";
+
+ JsonObject crdp_payload = new JsonObject();
+
+ crdp_payload.addProperty("protection_policy_name", protection_profile);
+
+ String sensitive = null;
+ OkHttpClient client = new OkHttpClient().newBuilder().build();
+ MediaType mediaType = MediaType.parse("application/json");
+ String urlStr = "http://" + crdpip + ":8090/v1/" + mode;
+
+ // encrypt data
+ for (int i = 0; i < snowflakedata.size(); i++) {
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ for (int j = 0; j < snowflakerow.size(); j++) {
+
+ if (j == 1) {
+ // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
+ // FPE example
+ sensitive = checkValid(snowflakerow);
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+
+ }
+ encdata = sensitive;
+
+ } else {
+
+ crdp_payload.addProperty(dataKey, sensitive);
+ if (mode.equals("reveal")) {
+ crdp_payload.addProperty("username", snowflakeuser);
+ if (keymetadatalocation.equalsIgnoreCase("external")) {
+ crdp_payload.addProperty("external_version", external_version_from_ext_source);
+ }
+ }
+ crdpjsonBody = crdp_payload.toString();
+ // System.out.println(crdpjsonBody);
+ RequestBody body = RequestBody.create(mediaType, crdpjsonBody);
+
+ Request crdp_request = new Request.Builder().url(urlStr).method("POST", body)
+ .addHeader("Content-Type", "application/json").build();
+ Response crdp_response = client.newCall(crdp_request).execute();
+
+ if (crdp_response.isSuccessful()) {
+
+ // Parse JSON response
+ String responseBody = crdp_response.body().string();
+ Gson gson = new Gson();
+ JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class);
+
+ if (jsonObject.has(jsonTagForProtectReveal)) {
+ protectedData = jsonObject.get(jsonTagForProtectReveal).getAsString();
+ if (keymetadatalocation.equalsIgnoreCase("external")
+ && mode.equalsIgnoreCase("protect")) {
+ externalkeymetadata = jsonObject.get("external_version").getAsString();
+ // System.out.println("Protected Data ext key metadata need to store this: "
+ // + externalkeymetadata);
+ }
+
+ if (keymetadatalocation.equalsIgnoreCase("internal")
+ && mode.equalsIgnoreCase("protect") && !showrevealkeybool) {
+ if (protectedData.length() > 7)
+ protectedData = protectedData.substring(7);
+ }
+
+ } else if (jsonObject.has("error_message")) {
+ String errorMessage = jsonObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ snowErrorMap.put(i, errorMessage);
+ bad_data = true;
+ } else
+ System.out.println("unexpected json value from results: ");
+ } else {
+ System.err.println("Request failed with status code: " + crdp_response.code());
+ }
+
+ crdp_response.close();
+
+ encdata = protectedData;
+
+ }
+
+ rownbranddata.add(encdata);
+ replies.add(rownbranddata);
+ rownbranddata = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ row_number = snowflakecolumn.getAsInt();
+ rownbranddata.add(row_number);
+
+ }
+
+ }
+
+ }
+
+ result.add("data", replies);
+ snowflakereturnstring = result.toString();
+
+ } catch (Exception e) {
+
+ System.out.println("in exception with " + e.getMessage());
+ snowflakereturnstring = "exception ";
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
+
+ result = new JsonObject();
+ replies = new JsonArray();
+ rownbranddata = new JsonArray();
+
+ for (int i = 0; i < snowflakedata.size(); i++) {
+
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ for (int j = 0; j < snowflakerow.size(); j++) {
+ if (j == 1) {
+ // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
+ // FPE example
+ String sensitive = checkValid(snowflakerow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ // System.out.println("adding null not charint or nbr");
+ sensitive = sensitive.replace("notvalid", "");
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null")
+ || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+
+ }
+ encdata = sensitive;
+
+ } else {
+ // System.out.println("normal number data" + sensitive);
+ }
+ rownbranddata.add(sensitive);
+ replies.add(rownbranddata);
+ rownbranddata = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ int row_number = snowflakecolumn.getAsInt();
+ rownbranddata.add(row_number);
+ }
+ }
+ }
+
+ result.add("data", replies);
+ JsonObject inputJsonObject = new JsonObject();
+ String bodyString = result.toString();
+ inputJsonObject.addProperty("statusCode", 200);
+ inputJsonObject.addProperty("body", bodyString);
+
+ snowflakereturnstring = inputJsonObject.toString();
+ // System.out.println(" new data " + snowflakereturnstring);
+
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+
+ } finally {
+
+ }
+ // System.out.println(snowflakereturnstring);
+ response.getWriter().write(snowflakereturnstring);
+
+ }
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer snowflakereturndatasb = new StringBuffer();
+
+ snowflakereturndatasb.append("{ \"statusCode\":");
+ snowflakereturndatasb.append(statusCode);
+ snowflakereturndatasb.append(",");
+ snowflakereturndatasb.append(" \"body\": {");
+ snowflakereturndatasb.append(" \"data\": [");
+ snowflakereturndatasb.append("] }}");
+ System.out.println("in exception with ");
+ return snowflakereturndatasb.toString();
+ }
+
+ public String checkValid(JsonArray snowrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (snowrow != null && snowrow.size() > 0) {
+ JsonElement element = snowrow.get(1);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+
+ boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
+
+ return founduserinuserset;
+
+ }
+
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/CMUserSetHelper.java b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/CMUserSetHelper.java
new file mode 100644
index 00000000..06f17015
--- /dev/null
+++ b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/CMUserSetHelper.java
@@ -0,0 +1,503 @@
+package com.example;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.security.KeyStore;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509TrustManager;
+/*
+ * This app provides a number of different helper methods dealing with CM Application Data Protection UserSets. There is a method to find
+ * a user in a userset and another method to populate the userset from a flat file. Usersets are typically used within
+ * an access policy but they are not restricted to that usage.
+ * Was tested with CM 2.14
+ * Note: This source code is only to be used for testing and proof of concepts.
+ * @author mwarner
+ *
+ */
+
+public class CMUserSetHelper {
+
+ static String hostnamevalidate = "yourhostname";
+
+ String usersetid = "716f01a6-5cab-4799-925a-6dc2d8712fc1";
+ String cmIP = "yourip";
+ String apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?name=";
+ // + username_in_userset;
+ String addusertouserset = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users";
+
+ int totalrecords = 0;
+ String authUrl = "https://" + cmIP + "/api/v1/auth/tokens";
+
+ static boolean debug = true;
+ int chunksize = 5;
+ static int CHUNKSIZEMAX = 100;
+
+ public CMUserSetHelper(String usersetid, String cmIP) {
+
+ this.usersetid = usersetid;
+ this.cmIP = cmIP;
+ this.apiUrlGetUsers = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users?name=";
+ this.addusertouserset = "https://" + cmIP + "/api/v1/data-protection/user-sets/" + usersetid + "/users";
+ this.authUrl = "https://" + cmIP + "/api/v1/auth/tokens";
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ String username = args[0];
+ String password = args[1];
+ String cmip = args[2];
+ String usersetid = args[3];
+ String filePath = args[4];
+ CMUserSetHelper cmusersetHelper = new CMUserSetHelper(usersetid, cmip);
+ int totalrecords = 0;
+
+ String jwthtoken = geAuthToken(cmusersetHelper.authUrl, username, password);
+
+ String newtoken = "Bearer " + removeQuotes(jwthtoken);
+
+ // String filePath =
+ // "C:\\Users\\t0185905\\workspace\\CT-VL-GCP\\src\\main\\java\\com\\example\\emailAddresses.txt";
+ RandomAccessFile file = new RandomAccessFile(filePath, "r");
+ if (cmusersetHelper.chunksize > CHUNKSIZEMAX)
+ cmusersetHelper.chunksize = CHUNKSIZEMAX;
+ int totoalnbrofrecords = numberOfLines(file);
+ if (cmusersetHelper.chunksize > totoalnbrofrecords) {
+ cmusersetHelper.chunksize = totoalnbrofrecords / 2;
+ }
+ //totalrecords = cmusersetHelper.addAUserToUserSet(cmusersetHelper.addusertouserset, newtoken);
+ totalrecords = cmusersetHelper.addAUserToUserSetFromFile(cmusersetHelper.addusertouserset, newtoken, filePath);
+ System.out.println("Totalrecords inserted into Userset " + cmusersetHelper.usersetid + " = " + totalrecords);
+
+ }
+
+ /**
+ * Returns an boolean if user found
+ *
+ *
+ * @param user user to find
+ * @param newtoken jwt token to use
+ * @return boolean true if found in userset
+ * @throws CustomException
+ */
+ public boolean findUserInUserSet(String user, String newtoken) throws CustomException {
+ boolean found = false;
+
+ String apiUrl = this.apiUrlGetUsers + user;
+
+ apiUrl = removeQuotes(apiUrl);
+
+ try {
+ URL url = new URL(apiUrl);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestMethod("GET");
+ connection.setRequestProperty("Authorization", newtoken);
+ connection.setRequestProperty("accept", "application/json");
+
+ connection.setDoInput(true);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+
+ StringBuilder response = new StringBuilder();
+ String line;
+ while ((line = reader.readLine()) != null) {
+ response.append(line);
+ }
+ reader.close();
+
+ Gson gson = new Gson();
+ if (debug)
+ System.out.println("response " + response);
+ JsonObject input = null;
+ JsonElement total = null;
+ JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ input = rootNode.getAsJsonObject();
+ if (input.isJsonObject()) {
+ total = input.get("total");
+ }
+ }
+ JsonPrimitive column = total.getAsJsonPrimitive();
+ String totalstr = column.getAsJsonPrimitive().toString();
+
+ Integer i = Integer.valueOf(totalstr);
+
+ if (i > 0) {
+ found = true;
+
+ }
+ connection.disconnect();
+ } catch (Exception e) {
+ if (e.getMessage().contains("403")) {
+ throw new CustomException("1002, User Not in Application Data Protection Clients ", 1002);
+ } else
+ e.printStackTrace();
+ }
+
+ return found;
+
+ }
+
+ /**
+ * Loads users from a file to the userset. Returns an int of number of users
+ * added.
+ *
+ *
+ * @param url url to userset api
+ * @param newtoken jwt token to use
+ * @param filePath file to load
+ * @return int totalnumberofrecords added to userset
+ */
+
+ public int addAUserToUserSetFromFile(String url, String newtoken, String filePath) throws IOException {
+
+ int totalnbrofrecords = 0;
+ try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
+ String line;
+ int count = 0;
+ StringBuilder payloadBuilder = new StringBuilder();
+
+ payloadBuilder.append("{\"users\": [");
+
+ while ((line = br.readLine()) != null) {
+ totalnbrofrecords++;
+ payloadBuilder.append("\"").append(line).append("\",");
+ count++;
+
+ // If 'n' records have been read, print the payload
+ if (count == chunksize) {
+ makeCMCall(payloadBuilder, newtoken, url);
+ // Reset payload builder and count for the next batch of records
+ payloadBuilder = new StringBuilder("{\"users\": [");
+ count = 0;
+ }
+ }
+
+ // If there are remaining records, print the payload
+ if (count > 0) {
+ payloadBuilder.deleteCharAt(payloadBuilder.length() - 1); // Remove the trailing comma
+ payloadBuilder.append("}");
+ makeCMCall(payloadBuilder, newtoken, url);
+ if (debug)
+ System.out.println(payloadBuilder.toString());
+ }
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ return totalnbrofrecords;
+
+ }
+
+ /**
+ * Loads users from a file to the userset. Returns an int of number of users
+ * added.
+ *
+ *
+ * @param payloadBuilder payload for CM call that contains users to add
+ * @param newtoken jwt token to use
+ * @param url url to peform inserts
+ * @return int response code
+ */
+
+ public static int makeCMCall(StringBuilder payloadBuilder, String newtoken, String url) throws IOException {
+
+ payloadBuilder.deleteCharAt(payloadBuilder.length() - 1); // Remove the trailing comma
+ payloadBuilder.append("]}");
+ if (debug)
+ System.out.println(payloadBuilder.toString());
+ String payload = payloadBuilder.toString();
+
+ // Create URL object
+ URL apiUrl = new URL(url);
+ // Create connection object
+ HttpURLConnection connection = (HttpURLConnection) apiUrl.openConnection();
+ connection.setRequestMethod("POST");
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setDoOutput(true);
+ connection.setRequestProperty("Authorization", newtoken);
+ // Send request
+ OutputStream outputStream = connection.getOutputStream();
+ outputStream.write(payload.getBytes());
+ outputStream.flush();
+ outputStream.close();
+
+ // Get response
+ int responseCode = connection.getResponseCode();
+ BufferedReader reader;
+ if (responseCode >= 200 && responseCode < 300) {
+ reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+ } else {
+ reader = new BufferedReader(new InputStreamReader(connection.getErrorStream()));
+ }
+
+ // Read response
+ String lineresponse;
+ StringBuilder response = new StringBuilder();
+ while ((lineresponse = reader.readLine()) != null) {
+ response.append(lineresponse);
+ }
+ reader.close();
+
+ // Print response
+ if (debug) {
+ System.out.println("Response Code: " + responseCode);
+ System.out.println("Response Body: " + response.toString());
+ }
+ // Disconnect connection
+ connection.disconnect();
+ // Reset payload builder and count for the next batch of records
+ payloadBuilder = new StringBuilder("{\"users\": [");
+
+ return responseCode;
+
+ }
+
+ /**
+ * Simple sample of showing how to load a couple of users to the userset.
+ * Returns an int of number of users added.
+ *
+ *
+ * @param url url to userset api
+ * @param newtoken jwt token to use
+ *
+ * @return int response code
+ */
+ public static int addAUserToUserSet(String url, String newtoken) throws IOException {
+ boolean found = false;
+
+ String payload = "{\"users\": [\"akhip@company.com\",\"user2@company.com\"]}";
+
+ // Create URL object
+ URL apiUrl = new URL(url);
+ // Create connection object
+ HttpURLConnection connection = (HttpURLConnection) apiUrl.openConnection();
+ connection.setRequestMethod("POST");
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setDoOutput(true);
+ connection.setRequestProperty("Authorization", newtoken);
+ // Send request
+ OutputStream outputStream = connection.getOutputStream();
+ outputStream.write(payload.getBytes());
+ outputStream.flush();
+ outputStream.close();
+
+ // Get response
+ int responseCode = connection.getResponseCode();
+ BufferedReader reader;
+ if (responseCode >= 200 && responseCode < 300) {
+ reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+ } else {
+ reader = new BufferedReader(new InputStreamReader(connection.getErrorStream()));
+ }
+
+ // Read response
+ String line;
+ StringBuilder response = new StringBuilder();
+ while ((line = reader.readLine()) != null) {
+ response.append(line);
+ }
+ reader.close();
+
+ // Print response
+ if (debug) {
+ System.out.println("Response Code: " + responseCode);
+ System.out.println("Response Body: " + response.toString());
+ }
+ // Disconnect connection
+ connection.disconnect();
+
+ return responseCode;
+
+ }
+
+ public static String removeQuotes(String input) {
+
+ // Remove double quotes from the input string
+ input = input.replace("\"", "");
+
+ return input;
+ }
+
+ private static int numberOfLines(RandomAccessFile file) throws IOException {
+ int numberOfLines = 0;
+ while (file.readLine() != null) {
+ numberOfLines++;
+ }
+ file.seek(0);
+ return numberOfLines;
+ }
+
+ /**
+ * Get JWT from CM
+ *
+ *
+ * @param apiUrl url to CM auth
+ * @param username username on CM
+ * @param pwd password on CM
+ * @return String jwt
+ */
+
+ public static String geAuthToken(String apiUrl, String usernb, String pwd) throws Exception
+
+ {
+
+ String jStr = "{\"username\":\"" + usernb + "\",\"password\":\"" + pwd + "\"}";
+ disableCertValidation();
+
+ String jwtstr = null;
+ try {
+ URL url = new URL(apiUrl);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestProperty("Content-length", String.valueOf(jStr.length()));
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setRequestMethod("GET");
+ connection.setDoOutput(true);
+ connection.setDoInput(true);
+ DataOutputStream output = new DataOutputStream(connection.getOutputStream());
+ output.writeBytes(jStr);
+ output.close();
+ BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+ StringBuilder response = new StringBuilder();
+ String line;
+ while ((line = reader.readLine()) != null) {
+ response.append(line);
+ }
+ reader.close();
+
+ JsonObject input = null;
+ JsonElement jwt = null;
+ JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ input = rootNode.getAsJsonObject();
+ if (input.isJsonObject()) {
+ jwt = input.get("jwt");
+ }
+ }
+ JsonPrimitive column = jwt.getAsJsonPrimitive();
+ jwtstr = column.getAsJsonPrimitive().toString();
+ connection.disconnect();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ return jwtstr;
+
+ }
+
+ /**
+ * Get JWT from CM
+ *
+ *
+ * @param apiUrl url to CM auth
+ * @param username username on CM
+ * @param pwd password on CM
+ * @return String jwt
+ */
+
+ public static String geAuthToken(String apiUrl) throws Exception
+
+ {
+
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+ String jStr = "{\"username\":\"" + userName + "\",\"password\":\"" + password + "\"}";
+
+ disableCertValidation();
+
+ String jwtstr = null;
+ try {
+ URL url = new URL(apiUrl);
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ connection.setRequestProperty("Content-length", String.valueOf(jStr.length()));
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setRequestMethod("GET");
+ connection.setDoOutput(true);
+ connection.setDoInput(true);
+ DataOutputStream output = new DataOutputStream(connection.getOutputStream());
+ output.writeBytes(jStr);
+ output.close();
+ BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+ StringBuilder response = new StringBuilder();
+ String line;
+ while ((line = reader.readLine()) != null) {
+ response.append(line);
+ }
+ reader.close();
+ if (debug)
+ System.out.println("response " + response);
+ JsonObject input = null;
+ JsonElement jwt = null;
+ JsonElement rootNode = JsonParser.parseString(response.toString()).getAsJsonObject();
+ if (rootNode.isJsonObject()) {
+ input = rootNode.getAsJsonObject();
+ if (input.isJsonObject()) {
+ jwt = input.get("jwt");
+ }
+ }
+ JsonPrimitive column = jwt.getAsJsonPrimitive();
+ jwtstr = column.getAsJsonPrimitive().toString();
+ connection.disconnect();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ return jwtstr;
+
+ }
+
+ public static void disableCertValidation() throws Exception {
+ TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() {
+ public java.security.cert.X509Certificate[] getAcceptedIssuers() {
+ return null;
+ }
+
+ public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) {
+ }
+
+ public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) {
+ }
+ } };
+
+ // Install the all-trusting trust manager
+ try {
+ SSLContext sc = SSLContext.getInstance("SSL");
+ sc.init(null, trustAllCerts, new java.security.SecureRandom());
+ HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ // Create all-trusting host name verifier
+ HostnameVerifier allHostsValid = new HostnameVerifier() {
+ public boolean verify(String hostname, SSLSession session) {
+ return true;
+ }
+ };
+
+ // Install the all-trusting host verifier
+ HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid);
+ }
+
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/CustomException.java b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/CustomException.java
new file mode 100644
index 00000000..44796974
--- /dev/null
+++ b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/CustomException.java
@@ -0,0 +1,15 @@
+
+
+
+class CustomException extends Exception {
+ private int errorCode;
+
+ public CustomException(String message, int errorCode) {
+ super(message);
+ this.errorCode = errorCode;
+ }
+
+ public int getErrorCode() {
+ return errorCode;
+ }
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCADPBulkFPETester.java b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCADPBulkFPETester.java
new file mode 100644
index 00000000..7beb3ef5
--- /dev/null
+++ b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCADPBulkFPETester.java
@@ -0,0 +1,525 @@
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+
+import com.google.cloud.functions.HttpFunction;
+import com.google.cloud.functions.HttpRequest;
+import com.google.cloud.functions.HttpResponse;
+import com.ingrian.security.nae.IngrianProvider;
+import com.ingrian.security.nae.AbstractNAECipher;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec;
+import com.ingrian.security.nae.NAEKey;
+import com.ingrian.security.nae.NAESession;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
+import com.ingrian.security.nae.IngrianProvider.Builder;
+import com.ingrian.security.nae.NAECipher;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonPrimitive;
+
+import java.math.BigInteger;
+import java.security.spec.AlgorithmParameterSpec;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Logger;
+
+/* This sample GCP Function is used to implement a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application
+ * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
+ * data so applications or business intelligence tools do not have to change in order to use these columns.
+*
+* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
+* for all possible data sizes and combinations of encryption algorithms and IV, etc.
+* Was tested with CM 2.14 & CADP 8.16.001
+* For more information on CADP see link below.
+https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html
+* For more information on Snowflake External Functions see link below.
+https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp
+
+Notes: This example uses the CADP bulk API.
+Maximum elements supported are 10000 and each data element must be of size <= 3500. If the data
+element has Unicode characters then the supported size limit would be 1750 characters
+
+Size of spec array should be same as the number of elements if user wants to use separate spec values
+for each data index. Spec array of size equal to data size is passed. Each spec array index represents corresponding data
+index.
+ *
+ *@author mwarner
+ *
+ */
+public class ThalesGCPSnowCADPBulkFPETester implements HttpFunction {
+// @Override
+ private static byte[][] data;
+ private static AlgorithmParameterSpec[] spec;
+ private static Integer[] snowrownbrs;
+
+ private static int BATCHLIMIT = 10000;
+ private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPBulkFPETester.class.getName());
+ private static final Gson gson = new Gson();
+ private static final String BADDATATAG = new String ("9999999999999999");
+
+ public static void main(String[] args) throws Exception {
+
+ ThalesGCPSnowCADPBulkFPETester nw2 = new ThalesGCPSnowCADPBulkFPETester();
+
+ String requestnull = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"\" ],\r\n" + " [ 1, \"null\" ]\r\n"
+ + " ]\r\n" + "}";
+
+ String request = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"pagetest\" ],\r\n" + " [1, \"11\" ],\r\n"
+ + " [2, \"0\" ],\r\n" + " [ 3, \"life, the universe, and everything\" ]\r\n" + " ]\r\n" + "}";
+
+ String requestnbr = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"23423453244\" ],\r\n"
+ + " [1, \"11\" ],\r\n" + " [2, \"000000\" ],\r\n" + " [ 3, \"4523048723948773883\" ]\r\n"
+ + " ]\r\n" + "}";
+
+ String protect_nbr102 = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"0001\" ],\r\n"
+ + " [1, \"0002\" ],\r\n" + " [2, \"0003\" ],\r\n" + " [3, \"0004\" ],\r\n"
+ + " [4, \"0005\" ],\r\n" + " [5, \"0006\" ],\r\n" + " [6, \"0007\" ],\r\n"
+ + " [7, \"0008\" ],\r\n" + " [8, \"0009\" ],\r\n" + " [9, \"0010\" ],\r\n"
+ + " [10, \"0011\" ],\r\n" + " [ 11, \"0012\" ]\r\n" + " ]\r\n" + "}";
+
+
+ String charint = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"2342-345-3244\" ],\r\n"
+ + " [1, \"1\" ],\r\n" + " [2, \"000-000\" ],\r\n"
+ + " [ 3, \"4523-048723-948773883\" ]\r\n" + " ]\r\n" + "}";
+
+ String charintdec = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"1030-236-1715\" ],\r\n"
+ + " [1, \"17042082183094609\" ],\r\n" + " [2, \"557-917\" ],\r\n"
+ + " [ 3, \"5078-969877-741002365\" ]\r\n" + " ]\r\n" + "}";
+
+ //snowflakereturnstring = {"data":[[0,"1030-236-1715"],[1,"17042082183094609"],[2,"557-917"],[3,"5078-969877-741002365"]]}
+
+ String response = null;
+ nw2.service(protect_nbr102, response);
+
+ }
+
+ public void service(String request, String response) throws Exception {
+
+ String tweakAlgo = null;
+ String tweakData = null;
+ String snowflakereturnstring = null;
+ JsonArray snowflakedata = null;
+ NAESession session = null;
+
+ Map encryptedErrorMapTotal = new HashMap();
+ JsonObject body = null;
+ int statusCode = 200;
+ int numberofchunks = 0;
+
+ String keyName = "testfaas";
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be
+ // returned when the user above does not have access to the key and if doing a lookup in the userset
+ // and the user does not exist. If returnciphertextforuserwithnokeyaccess = no then an error will be
+ // returned to the query, else the results set will provide ciphertext.
+ // yes/no
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB?
+ // yes/no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it
+ // is the userset in CM but could be a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ String datatype = System.getenv("datatype");
+ // mode of operation valid values are : encrypt or decrypt
+ String mode = System.getenv("mode");
+
+ int cipherType = 0;
+ if (mode.equals("encrypt"))
+ cipherType = Cipher.ENCRYPT_MODE;
+ else
+ cipherType = Cipher.DECRYPT_MODE;
+
+ int batchsize = Integer.parseInt(System.getenv("BATCHSIZE"));
+
+ try {
+
+ // This code is only to be used when input data contains user info.
+ /*
+ * if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group
+ *
+ * boolean founduserinuserset = findUserInUserSet(bigquerysessionUser, userName, password, usersetID,
+ * userSetLookupIP); // System.out.println("Found User " + founduserinuserset); if (!founduserinuserset)
+ * throw new CustomException("1001, User Not in User Set", 1001);
+ *
+ * } else { usersetlookupbool = false; }
+ */
+ // How many records in a chunk. Testing has indicated point of diminishing returns at 100 or 200, but
+ // may vary depending on size of data.
+
+ try {
+ JsonElement requestParsed = gson.fromJson(request, JsonElement.class);
+ JsonObject requestJson = null;
+
+ if (requestParsed != null && requestParsed.isJsonObject()) {
+ requestJson = requestParsed.getAsJsonObject();
+ }
+
+ if (requestJson != null && requestJson.has("data")) {
+ snowflakedata = requestJson.getAsJsonArray("data");
+
+ }
+
+ } catch (JsonParseException e) {
+ logger.severe("Error parsing JSON: " + e.getMessage());
+ }
+
+ System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
+ "D:\\product\\Build\\CADP_for_JAVA.properties");
+
+ session = NAESession.getSession(userName, password.toCharArray());
+ NAEKey key = NAEKey.getSecretKey(keyName, session);
+ int row_number = 0;
+
+ StringBuffer snowflakereturndata = new StringBuffer();
+ int numberOfLines = snowflakedata.size();
+ int totalRowsLeft = numberOfLines;
+
+ if (batchsize > numberOfLines)
+ batchsize = numberOfLines;
+ if (batchsize >= BATCHLIMIT)
+ batchsize = BATCHLIMIT;
+ spec = new FPEParameterAndFormatSpec[batchsize];
+ data = new byte[batchsize][];
+ snowrownbrs = new Integer[batchsize];
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+ JsonArray innerDataArray = new JsonArray();
+
+ String algorithm = null;
+
+ FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
+ .build();
+
+ if (datatype.equals("char"))
+ algorithm = "FPE/FF1/CARD62";
+ else if (datatype.equals("charint"))
+ algorithm = "FPE/FF1/CARD10";
+ else
+ algorithm = "FPE/FF1/CARD10";
+
+ AbstractNAECipher thalesCipher = NAECipher.getInstanceForBulkData(algorithm, "IngrianProvider");
+ int i = 0;
+
+ if (batchsize > numberOfLines)
+ batchsize = numberOfLines;
+ if (batchsize >= BATCHLIMIT)
+ batchsize = BATCHLIMIT;
+
+ spec = new FPEParameterAndFormatSpec[batchsize];
+ data = new byte[batchsize][];
+ snowrownbrs = new Integer[batchsize];
+ String sensitive = null;
+ thalesCipher.init(cipherType, key, spec[0]);
+
+ // int i = 0;
+ int count = 0;
+ boolean newchunk = true;
+ int dataIndex = 0;
+ int specIndex = 0;
+ int snowRowIndex = 0;
+ // String sensitive = null;
+
+ while (i < numberOfLines) {
+ int index = 0;
+
+ if (newchunk) {
+
+ if (totalRowsLeft < batchsize) {
+ spec = new FPEParameterAndFormatSpec[totalRowsLeft];
+ data = new byte[totalRowsLeft][];
+ snowrownbrs = new Integer[totalRowsLeft];
+ } else {
+ spec = new FPEParameterAndFormatSpec[batchsize];
+ data = new byte[batchsize][];
+ snowrownbrs = new Integer[batchsize];
+ }
+ newchunk = false;
+ }
+
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ for (int j = 0; j < snowflakerow.size(); j++) {
+
+ if (j == 1) {
+
+ // FPE example
+ sensitive = checkValid(snowflakerow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ sensitive = BADDATATAG+sensitive;
+ data[dataIndex++] = sensitive.getBytes();
+ // Can not return number since a leading 0 will not work.
+ // innerDataArray.add(new BigInteger(sensitive));
+ } else
+ data[dataIndex++] = BADDATATAG.getBytes();
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+ data[dataIndex++] = sensitive.getBytes();
+ } else {
+ data[dataIndex++] = sensitive.getBytes();
+ }
+ spec[specIndex++] = param;
+
+ System.out.println("not valid or null");
+ } else {
+ data[dataIndex++] = sensitive.getBytes();
+ spec[specIndex++] = param;
+ }
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ row_number = snowflakecolumn.getAsInt();
+ snowrownbrs[snowRowIndex++] = row_number;
+
+ }
+
+ }
+
+ if (count == batchsize - 1) {
+
+ // Map to store exceptions while encryption
+ Map encryptedErrorMap = new HashMap();
+
+ byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap);
+
+ for (Map.Entry entry : encryptedErrorMap.entrySet()) {
+ Integer mkey = entry.getKey();
+ String mvalue = entry.getValue();
+ encryptedErrorMapTotal.put(mkey, mvalue);
+ }
+
+ for (int enc = 0; enc < encryptedData.length; enc++) {
+
+ innerDataArray.add(snowrownbrs[enc]);
+ innerDataArray.add(new String(encryptedData[enc]));
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ index++;
+
+ }
+
+ numberofchunks++;
+ newchunk = true;
+ count = 0;
+ dataIndex = 0;
+ specIndex = 0;
+ snowRowIndex = 0;
+ } else
+ count++;
+
+ totalRowsLeft--;
+ i++;
+ }
+ if (count > 0) {
+ numberofchunks++;
+ int index = 0;
+ Map encryptedErrorMap = new HashMap();
+ byte[][] encryptedData = thalesCipher.doFinalBulk(data, spec, encryptedErrorMap);
+ for (int enc = 0; enc < encryptedData.length; enc++) {
+ innerDataArray.add(snowrownbrs[enc]);
+ innerDataArray.add(new String(encryptedData[enc]));
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ index++;
+
+ }
+ }
+
+ bodyObject.add("data", dataArray);
+ String bodyString = bodyObject.toString();
+
+ snowflakereturnstring = bodyString.toString();
+
+ } catch (
+
+ Exception e) {
+ System.out.println("in exception with " + e.getMessage());
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
+
+ try {
+ snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, true, null, datatype);
+ } catch (IllegalBlockSizeException e1) {
+ // TODO Auto-generated catch block
+ e1.printStackTrace();
+ } catch (BadPaddingException e1) {
+ // TODO Auto-generated catch block
+ e1.printStackTrace();
+ }
+
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } finally {
+ if (session != null) {
+ session.closeSession();
+ }
+ }
+ System.out.println("number of chunks = " + numberofchunks);
+ System.out.println("snowflakereturnstring = " + snowflakereturnstring);
+ // response.getWriter().write(snowflakereturnstring);
+
+ }
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer snowflakereturndatasb = new StringBuffer();
+
+ snowflakereturndatasb.append("{ \"statusCode\":");
+ snowflakereturndatasb.append(statusCode);
+ snowflakereturndatasb.append(",");
+ snowflakereturndatasb.append(" \"body\": {");
+ snowflakereturndatasb.append(" \"data\": [");
+ snowflakereturndatasb.append("] }}");
+ System.out.println("in exception with ");
+ return snowflakereturndatasb.toString();
+ }
+
+ public String checkValid(JsonArray snowrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (snowrow != null && snowrow.size() > 0) {
+ JsonElement element = snowrow.get(1);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+
+ public String formatReturnValue(int statusCode, JsonArray snowflakedata, boolean error, Cipher thalesCipher,
+ String datatype) throws IllegalBlockSizeException, BadPaddingException {
+ int row_number = 0;
+
+ String encdata = null;
+ String sensitive = null;
+
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+ JsonArray innerDataArray = new JsonArray();
+
+ for (int i = 0; i < snowflakedata.size(); i++) {
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+ for (int j = 0; j < snowflakerow.size(); j++) {
+ if (j == 1) {
+ sensitive = checkValid(snowflakerow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ System.out.println("adding null not charint or nbr");
+ sensitive = sensitive.replace("notvalid", "");
+ innerDataArray.add(sensitive);
+ // Can not return number since a leading 0 will not work.
+ // innerDataArray.add(new BigInteger(sensitive));
+ } else
+ innerDataArray.add(BADDATATAG);
+
+ System.out.println("datatype charint or nbr adding big int 9999999999999999");
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+ innerDataArray.add("");
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ innerDataArray.add(sensitive);
+ } else {
+ innerDataArray.add(sensitive);
+ }
+
+ } else {
+ if (!error) {
+ byte[] outbuf = thalesCipher.doFinal(sensitive.getBytes());
+ encdata = new String(outbuf);
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ innerDataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ innerDataArray.add(encdata);
+ }
+ } else {
+ encdata = sensitive;
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ innerDataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ innerDataArray.add(encdata);
+ }
+ }
+ }
+
+ // innerDataArray.add(encdata);
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ row_number = snowflakecolumn.getAsInt();
+ innerDataArray.add(row_number);
+ }
+ }
+ }
+
+ bodyObject.add("data", dataArray);
+ String bodyString = bodyObject.toString();
+
+ System.out.println("bodystr " + bodyString);
+
+ return bodyString;
+
+ }
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+
+ boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
+
+ return founduserinuserset;
+
+ }
+
+ @Override
+ public void service(HttpRequest request, HttpResponse response) throws Exception {
+ // TODO Auto-generated method stub
+
+ }
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCADPFPETester.java b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCADPFPETester.java
new file mode 100644
index 00000000..d1bb677e
--- /dev/null
+++ b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCADPFPETester.java
@@ -0,0 +1,337 @@
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+
+import com.google.cloud.functions.HttpFunction;
+import com.google.cloud.functions.HttpRequest;
+import com.google.cloud.functions.HttpResponse;
+import com.ingrian.security.nae.IngrianProvider;
+import com.ingrian.security.nae.FPECharset;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec;
+import com.ingrian.security.nae.NAEKey;
+import com.ingrian.security.nae.NAESession;
+import com.ingrian.security.nae.FPEParameterAndFormatSpec.FPEParameterAndFormatBuilder;
+import com.ingrian.security.nae.IngrianProvider.Builder;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonPrimitive;
+
+import java.math.BigInteger;
+import java.util.logging.Logger;
+
+/* This test app to test the logic for a Snowflake Database User Defined Function(UDF). It is an example of how to use Thales Cipher Trust Manager Protect Application
+ * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
+ * data so applications or business intelligence tools do not have to change in order to use these columns.
+*
+* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
+* for all possible data sizes and combinations of encryption algorithms and IV, etc.
+* Was tested with CM 2.14 & CADP 8.16
+* For more information on CADP see link below.
+https://thalesdocs.com/ctp/con/cadp/cadp-java/latest/admin/index.html
+* For more information on Snowflake External Functions see link below.
+https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp
+ *
+ *@author mwarner
+ *
+ */
+public class ThalesGCPSnowCADPFPETester implements HttpFunction {
+// @Override
+
+ private static final Logger logger = Logger.getLogger(ThalesGCPSnowCADPFPETester.class.getName());
+ private static final Gson gson = new Gson();
+ private static final String BADDATATAG = new String ("9999999999999999");
+
+ public static void main(String[] args) throws Exception {
+
+ ThalesGCPSnowCADPFPETester nw2 = new ThalesGCPSnowCADPFPETester();
+
+ String requestnull = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"\" ],\r\n" + " [ 1, \"null\" ]\r\n"
+ + " ]\r\n" + "}";
+
+ String request = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"pagetest\" ],\r\n" + " [1, \"11\" ],\r\n"
+ + " [2, \"0\" ],\r\n" + " [ 3, \"life, the universe, and everything\" ]\r\n" + " ]\r\n" + "}";
+
+ String requestnbr = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"342342348834\" ],\r\n"
+ + " [1, \"56\" ],\r\n" + " [2, \"0\" ],\r\n" + " [ 3, \"452345778833333\" ]\r\n"
+ + " ]\r\n" + "}";
+
+ String requestnbrinvalid = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"342342348834\" ],\r\n"
+ + " [1, \"null\" ],\r\n" + " [2, \"0\" ],\r\n" + " [ 3, \"452345778833333\" ]\r\n"
+ + " ]\r\n" + "}";
+
+ String response = null;
+ nw2.service(requestnbr, response);
+
+ }
+
+ public void service(String request, String response) throws Exception {
+
+ String encdata = "";
+ String tweakAlgo = null;
+ String tweakData = null;
+ String snowflakereturnstring = null;
+ JsonArray snowflakedata = null;
+ int statusCode = 200;
+
+ NAESession session = null;
+ String keyName = "testfaas";
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned
+ // when the user above does not have access to the key and if doing a
+ // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no
+ // then an error will be returned to the query, else the results set will provide ciphertext.
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ // yes,no
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB
+ // yes,no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be
+ // a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ String datatype = System.getenv("datatype");
+ // mode of operation valid values are : encrypt or decrypt
+ String mode = System.getenv("mode");
+
+ int cipherType = 0;
+ if (mode.equals("encrypt"))
+ cipherType = Cipher.ENCRYPT_MODE;
+ else
+ cipherType = Cipher.DECRYPT_MODE;
+
+ try {
+
+ // This code is only to be used when input data contains user info.
+ /*
+ * if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group
+ *
+ * boolean founduserinuserset = findUserInUserSet(bigquerysessionUser, userName, password, usersetID,
+ * userSetLookupIP); // System.out.println("Found User " + founduserinuserset); if (!founduserinuserset)
+ * throw new CustomException("1001, User Not in User Set", 1001);
+ *
+ * } else { usersetlookupbool = false; }
+ */
+
+ try {
+ JsonElement requestParsed = gson.fromJson(request, JsonElement.class);
+ JsonObject requestJson = null;
+
+ if (requestParsed != null && requestParsed.isJsonObject()) {
+ requestJson = requestParsed.getAsJsonObject();
+ }
+
+ if (requestJson != null && requestJson.has("data")) {
+ snowflakedata = requestJson.getAsJsonArray("data");
+
+ }
+
+ } catch (JsonParseException e) {
+ logger.severe("Error parsing JSON: " + e.getMessage());
+ }
+
+ /*
+ * System.setProperty( "com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
+ * "CADP_for_JAVA.properties"); IngrianProvider builder = new Builder().addConfigFileInputStream(
+ * getClass().getClassLoader().getResourceAsStream("CADP_for_JAVA.properties")). build();
+ */
+
+ System.setProperty("com.ingrian.security.nae.CADP_for_JAVA_Properties_Conf_Filename",
+ "D:\\product\\Build\\CADP_for_JAVA.properties");
+
+ session = NAESession.getSession(userName, password.toCharArray());
+ NAEKey key = NAEKey.getSecretKey(keyName, session);
+ int row_number = 0;
+
+ String algorithm = null;
+
+ FPEParameterAndFormatSpec param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo).build();
+ if (datatype.equals("char"))
+ {
+ algorithm = "FPE/FF1/CARD62";
+ }
+ else {
+ algorithm = "FPE/FF1/CARD10";
+ }
+
+ Cipher thalesCipher = Cipher.getInstance(algorithm, "IngrianProvider");
+ // initialize cipher to encrypt.
+ thalesCipher.init(cipherType, key, param);
+
+ snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, false, thalesCipher, datatype);
+
+ } catch (Exception e) {
+
+ System.out.println("in exception with " + e.getMessage());
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
+
+ snowflakereturnstring = formatReturnValue(statusCode, snowflakedata, true, null, datatype);
+
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+
+ } finally {
+ if (session != null) {
+ session.closeSession();
+ }
+ }
+ System.out.println(snowflakereturnstring);
+ // response.getWriter().write(snowflakereturnstring);
+
+ }
+
+ public String checkValid(JsonArray snowrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (snowrow != null && snowrow.size() > 0) {
+ JsonElement element = snowrow.get(1);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer snowflakereturndatasb = new StringBuffer();
+
+ snowflakereturndatasb.append("{ \"statusCode\":");
+ snowflakereturndatasb.append(statusCode);
+ snowflakereturndatasb.append(",");
+ snowflakereturndatasb.append(" \"body\": {");
+ snowflakereturndatasb.append(" \"data\": [");
+ snowflakereturndatasb.append("] }}");
+ System.out.println("in exception with ");
+ return snowflakereturndatasb.toString();
+ }
+
+ public String formatReturnValue(int statusCode, JsonArray snowflakedata, boolean error, Cipher thalesCipher,
+ String datatype) throws IllegalBlockSizeException, BadPaddingException {
+ int row_number = 0;
+
+ String encdata = null;
+ String sensitive = null;
+
+ JsonObject bodyObject = new JsonObject();
+ JsonArray dataArray = new JsonArray();
+ JsonArray innerDataArray = new JsonArray();
+
+ for (int i = 0; i < snowflakedata.size(); i++) {
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+ for (int j = 0; j < snowflakerow.size(); j++) {
+ if (j == 1) {
+ sensitive = checkValid(snowflakerow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ System.out.println("adding null not charint or nbr");
+ sensitive = sensitive.replace("notvalid", "");
+ innerDataArray.add(sensitive);
+ // Can not return number since a leading 0 will not work.
+ // innerDataArray.add(new BigInteger(sensitive));
+ } else
+ innerDataArray.add(BADDATATAG);
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+ innerDataArray.add("");
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ innerDataArray.add(sensitive);
+ } else {
+ innerDataArray.add(sensitive);
+ }
+
+ } else {
+ if (!error) {
+ byte[] outbuf = thalesCipher.doFinal(sensitive.getBytes());
+ encdata = new String(outbuf);
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ innerDataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ innerDataArray.add(encdata);
+ }
+ } else {
+ encdata = sensitive;
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ innerDataArray.add(encdata);
+ // innerDataArray.add(new BigInteger(encdata));
+ } else {
+ innerDataArray.add(encdata);
+ }
+ }
+ }
+
+ // innerDataArray.add(encdata);
+ dataArray.add(innerDataArray);
+ innerDataArray = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ row_number = snowflakecolumn.getAsInt();
+ innerDataArray.add(row_number);
+ }
+ }
+ }
+
+ bodyObject.add("data", dataArray);
+ String bodyString = bodyObject.toString();
+
+ System.out.println("bodystr " + bodyString);
+
+ return bodyString;
+
+ }
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+
+ boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
+
+ return founduserinuserset;
+
+ }
+
+ @Override
+ public void service(HttpRequest request, HttpResponse response) throws Exception {
+ // TODO Auto-generated method stub
+
+ }
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCRDPBulkFPETester.java b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCRDPBulkFPETester.java
new file mode 100644
index 00000000..c92ee948
--- /dev/null
+++ b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCRDPBulkFPETester.java
@@ -0,0 +1,588 @@
+
+import com.google.cloud.functions.HttpFunction;
+import com.google.cloud.functions.HttpRequest;
+import com.google.cloud.functions.HttpResponse;
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonPrimitive;
+
+import okhttp3.MediaType;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import java.util.logging.Logger;
+
+/* This test app to test the logic for a Snowflake Database User Defined Function(UDF).
+ * It is an example of how to use Thales CipherTrust REST Application Dataprotection (CRDP)
+ * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
+ * data so applications or business intelligence tools do not have to change in order to use these columns.
+*
+* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
+* for all possible data sizes and combinations of encryption algorithms and IV, etc.
+* Was tested with CM 2.14 and CRDP 1.0
+* For more information on CRDP see link below.
+https://thalesdocs.com/ctp/con/crdp/latest/index.html
+* For more information on Snowflake External Functions see link below.
+https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp
+ *
+ *@author mwarner
+ *
+ */
+public class ThalesGCPSnowCRDPBulkFPETester implements HttpFunction {
+// @Override
+
+ private static final Gson gson = new Gson();
+ private static final String BADDATATAG = new String("9999999999999999");
+ private static int BATCHLIMIT = 10000;
+
+ private static final String REVEALRETURNTAG = new String("data");
+ private static final String PROTECTRETURNTAG = new String("protected_data");
+
+ public static void main(String[] args) throws Exception {
+
+ ThalesGCPSnowCRDPBulkFPETester nw2 = new ThalesGCPSnowCRDPBulkFPETester();
+
+ String protectnull = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"null\" ],\r\n" + " [1, \"1\" ],\r\n"
+ + " [ 2, \"\" ]\r\n" + " ]\r\n" + "}";
+
+ String revealnull = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"9500405729277875\" ],\r\n"
+ + " [1, \"83724488316877864\" ],\r\n" + " [ 2, \"9500405729277875\" ]\r\n" + " ]\r\n" + "}";
+ // 1002001 {"data":[[0,"9500405729277875"],[1,"83724488316877864"],[2,"9500405729277875"]]}
+ String protect = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"pagetest\" ],\r\n"
+ + " [1, \"thisismoredata\" ],\r\n" + " [2, \"8989977666\" ],\r\n"
+ + " [ 3, \"life, the universe, and everything\" ]\r\n" + " ]\r\n" + "}";
+
+ String protect_nbr = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"234242344\" ],\r\n"
+ + " [1, \"6578-5545-7567\" ],\r\n" + " [2, \"3434\" ],\r\n" + " [ 3, \"333\" ]\r\n"
+ + " ]\r\n" + "}";
+
+ String reveal_ext_nbr = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"652425219\" ],\r\n"
+ + " [1, \"6684-5327-7937\" ],\r\n" + " [2, \"5295\" ],\r\n" + " [ 3, \"856\" ]\r\n"
+ + " ]\r\n" + "}";
+
+ String protect_nbr102 = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"0001\" ],\r\n"
+ + " [1, \"0002\" ],\r\n" + " [2, \"0003\" ],\r\n" + " [3, \"0004\" ],\r\n"
+ + " [4, \"0005\" ],\r\n" + " [5, \"0006\" ],\r\n" + " [6, \"0007\" ],\r\n"
+ + " [7, \"0008\" ],\r\n" + " [8, \"0009\" ],\r\n" + " [9, \"0010\" ],\r\n"
+ + " [10, \"0011\" ],\r\n" + " [ 11, \"0012\" ]\r\n" + " ]\r\n" + "}";
+
+ String reveal_ext_alpha = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"wSf2fghX\" ],\r\n"
+ + " [1, \"7nGV82KYDX001F\" ],\r\n" + " [2, \"GsrAUKzfje\" ],\r\n"
+ + " [ 3, \"Qqce, QBv d7ErVAPM, teq ZD6sjHPecT\" ]\r\n" + " ]\r\n" + "}";
+
+ // String reveal_ext_nbr = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"073675208\" ],\r\n"
+ // + " [1, \"5336-6160-6623\" ],\r\n" + " [2, \"1348\" ],\r\n" + " [ 3, \"054\" ]\r\n"
+ // + " ]\r\n" + "}";
+
+//temp{"data":[[0,"073675208"],[1,"5336-6160-6623"],[2,"1348"],[3,"054"]]}
+
+ String response = null;
+ nw2.service(protect_nbr102, response);
+
+ }
+
+ public void service(String request, String response) throws Exception {
+ Map bqErrorMap = new HashMap();
+ String encdata = "";
+ int error_count = 0;
+ int statusCode = 200;
+
+ JsonObject result = new JsonObject();
+ JsonArray replies = new JsonArray();
+ JsonArray rownbranddata = new JsonArray();
+
+ String keyName = "testfaas";
+ String crdpip = System.getenv("CRDPIP");
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned
+ // when the user above does not have access to the key and if doing a
+ // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no
+ // then an error will be returned to the query, else the results set will provide ciphertext.
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ // yes,no
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB
+ // yes,no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be
+ // a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ String keymetadatalocation = System.getenv("keymetadatalocation");
+ String external_version_from_ext_source = System.getenv("keymetadata");
+ String protection_profile = System.getenv("protection_profile");
+ String mode = System.getenv("mode");
+ String datatype = System.getenv("datatype");
+
+ String inputDataKey = null;
+ String outputDataKey = null;
+ String protectedData = null;
+ String externalkeymetadata = null;
+ String jsonBody = null;
+ String jsonTagForProtectReveal = null;
+
+ boolean bad_data = false;
+
+ String showrevealkey = "yes";
+
+ if (mode.equals("protectbulk")) {
+ inputDataKey = "data_array";
+ outputDataKey = "protected_data_array";
+ jsonTagForProtectReveal = PROTECTRETURNTAG;
+ if (keymetadatalocation.equalsIgnoreCase("internal")) {
+ showrevealkey = System.getenv("showrevealinternalkey");
+ if (showrevealkey == null)
+ showrevealkey = "yes";
+ }
+ } else {
+ inputDataKey = "protected_data_array";
+ outputDataKey = "data_array";
+ jsonTagForProtectReveal = REVEALRETURNTAG;
+ }
+
+ boolean showrevealkeybool = showrevealkey.equalsIgnoreCase("yes");
+
+ String snowflakeuser = "snowflakeuser";
+ JsonArray snowflakedata = null;
+ String snowflakereturnstring = null;
+ Response crdp_response = null;
+ try {
+
+ // This code is only to be used when input data contains user info.
+ /*
+ * if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group
+ *
+ * boolean founduserinuserset = findUserInUserSet(bigquerysessionUser, userName, password, usersetID,
+ * userSetLookupIP); // System.out.println("Found User " + founduserinuserset); if (!founduserinuserset)
+ * throw new CustomException("1001, User Not in User Set", 1001);
+ *
+ * } else { usersetlookupbool = false; }
+ */
+
+ try {
+ JsonElement requestParsed = gson.fromJson(request, JsonElement.class);
+ JsonObject requestJson = null;
+
+ if (requestParsed != null && requestParsed.isJsonObject()) {
+ requestJson = requestParsed.getAsJsonObject();
+ }
+
+ if (requestJson != null && requestJson.has("data")) {
+ snowflakedata = requestJson.getAsJsonArray("data");
+
+ }
+
+ } catch (JsonParseException e) {
+ System.out.println("Error parsing JSON: " + e.getMessage());
+ }
+
+ int row_number = 0;
+
+ // Serialization
+
+ int numberofchunks = 0;
+ StringBuffer protection_policy_buff = new StringBuffer();
+ String notvalid = "notvalid";
+ // StringBuffer snowflakereturndata = new StringBuffer();
+
+ int numberOfLines = snowflakedata.size();
+ int totalRowsLeft = numberOfLines;
+ // int batchsize = Integer.parseInt(System.getenv("BATCHSIZE"));
+ int batchsize = 10;
+ if (batchsize > numberOfLines)
+ batchsize = numberOfLines;
+ if (batchsize >= BATCHLIMIT)
+ batchsize = BATCHLIMIT;
+ // Serialization
+
+ String crdpjsonBody = null;
+ // String external_version_from_ext_source = "1004001";
+ // String external_version_from_ext_source = "1001001";
+
+ int i = 0;
+ int count = 0;
+ int totalcount = 0;
+ int dataIndex = 0; // assumes index from snowflake will always be sequential.
+ JsonObject crdp_payload = new JsonObject();
+ String sensitive = null;
+ JsonArray crdp_payload_array = new JsonArray();
+
+ OkHttpClient client = new OkHttpClient().newBuilder().build();
+ MediaType mediaType = MediaType.parse("application/json");
+ String urlStr = "http://" + crdpip + ":8090/v1/" + mode;
+
+ while (i < numberOfLines) {
+
+ for (int b = 0; b < batchsize && b < totalRowsLeft; b++) {
+
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ sensitive = checkValid(snowflakerow);
+
+ protection_profile = protection_profile.trim();
+ // Format the output
+ String formattedElement = String.format("\"protection_policy_name\" : \"%s\"", protection_profile);
+ protection_policy_buff.append(formattedElement);
+ protection_policy_buff.append(",");
+
+ if (mode.equals("protectbulk")) {
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ // System.out.println("adding null not charint or nbr");
+ sensitive = sensitive.replace("notvalid", "");
+ sensitive = BADDATATAG + sensitive;
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ // sensitive = sensitive.replace("notvalid", "");
+
+ }
+ encdata = sensitive;
+
+ }
+ crdp_payload_array.add(sensitive);
+ } else {
+ JsonObject protectedDataObject = new JsonObject();
+ protectedDataObject.addProperty(PROTECTRETURNTAG, sensitive);
+ if (keymetadatalocation.equalsIgnoreCase("external")) {
+ protectedDataObject.addProperty("external_version", external_version_from_ext_source);
+ }
+ crdp_payload_array.add(protectedDataObject);
+
+ }
+
+ if (count == batchsize - 1) {
+ crdp_payload.add(inputDataKey, crdp_payload_array);
+ String inputdataarray = null;
+ if (mode.equals("revealbulk")) {
+ crdp_payload.addProperty("username", snowflakeuser);
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ jsonBody = protection_policy_buff.toString();
+ jsonBody = jsonBody.replaceFirst("\\{", " ");
+
+ } else {
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ inputdataarray = protection_policy_buff.toString();
+ jsonBody = inputdataarray.replace("{", " ");
+ }
+ jsonBody = "{" + jsonBody;
+
+ RequestBody body = RequestBody.create(mediaType, jsonBody);
+
+ Request crdp_request = new Request.Builder().url(urlStr).method("POST", body)
+ .addHeader("Content-Type", "application/json").build();
+ crdp_response = client.newCall(crdp_request).execute();
+ String crdpreturnstr = null;
+ if (crdp_response.isSuccessful()) {
+ // Parse JSON response
+ String responseBody = crdp_response.body().string();
+ JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class);
+ JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey);
+
+ String status = jsonObject.get("status").getAsString();
+ int success_count = jsonObject.get("success_count").getAsInt();
+ error_count = jsonObject.get("error_count").getAsInt();
+ if (error_count > 0)
+ System.out.println("errors " + error_count);
+
+ for (JsonElement element : protectedDataArray) {
+
+ JsonObject protectedDataObject = element.getAsJsonObject();
+ if (protectedDataObject.has(jsonTagForProtectReveal)) {
+
+ protectedData = protectedDataObject.get(jsonTagForProtectReveal).getAsString();
+
+ if (keymetadatalocation.equalsIgnoreCase("internal")
+ && mode.equalsIgnoreCase("protectbulk") && !showrevealkeybool) {
+ if (protectedData.length() > 7)
+ protectedData = protectedData.substring(7);
+ }
+ System.out.println(protectedData);
+ rownbranddata.add(dataIndex);
+ rownbranddata.add(new String(protectedData));
+ replies.add(rownbranddata);
+ rownbranddata = new JsonArray();
+ if (mode.equals("protectbulk")) {
+ if (keymetadatalocation.equalsIgnoreCase("external")
+ && mode.equalsIgnoreCase("protectbulk")) {
+ externalkeymetadata = protectedDataObject.get("external_version")
+ .getAsString();
+ System.out.println("Protected Data ext key metadata need to store this: "
+ + externalkeymetadata);
+
+ }
+ }
+ } else if (protectedDataObject.has("error_message")) {
+ String errorMessage = protectedDataObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ bqErrorMap.put(i, errorMessage);
+ bad_data = true;
+ } else
+ System.out.println("unexpected json value from results: ");
+ dataIndex++;
+
+ }
+
+ crdp_payload_array = new JsonArray();
+ protection_policy_buff = new StringBuffer();
+ numberofchunks++;
+ totalcount = totalcount + count;
+ count = 0;
+ } else {// throw error....
+ System.err.println("Request failed with status code: " + crdp_response.code());
+ throw new CustomException("1010, Unexpected Error ", 1010);
+ }
+
+ } else {
+ count++;
+ }
+ totalRowsLeft--;
+ i++;
+ }
+ }
+ if (count > 0) {
+ crdp_payload.add(inputDataKey, crdp_payload_array);
+ String inputdataarray = null;
+ if (mode.equals("revealbulk")) {
+ crdp_payload.addProperty("username", snowflakeuser);
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ jsonBody = protection_policy_buff.toString();
+ jsonBody = jsonBody.replaceFirst("\\{", " ");
+
+ } else {
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ inputdataarray = protection_policy_buff.toString();
+ jsonBody = inputdataarray.replace("{", " ");
+ }
+ jsonBody = "{" + jsonBody;
+
+ RequestBody body = RequestBody.create(mediaType, jsonBody);
+
+ Request crdp_request = new Request.Builder().url(urlStr).method("POST", body)
+ .addHeader("Content-Type", "application/json").build();
+ crdp_response = client.newCall(crdp_request).execute();
+ String crdpreturnstr = null;
+ if (crdp_response.isSuccessful()) {
+ // Parse JSON response
+ String responseBody = crdp_response.body().string();
+ JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class);
+ JsonArray protectedDataArray = jsonObject.getAsJsonArray(outputDataKey);
+
+ String status = jsonObject.get("status").getAsString();
+ int success_count = jsonObject.get("success_count").getAsInt();
+ error_count = jsonObject.get("error_count").getAsInt();
+ if (error_count > 0)
+ System.out.println("errors " + error_count);
+
+ for (JsonElement element : protectedDataArray) {
+
+ JsonObject protectedDataObject = element.getAsJsonObject();
+ if (protectedDataObject.has(jsonTagForProtectReveal)) {
+
+ protectedData = protectedDataObject.get(jsonTagForProtectReveal).getAsString();
+
+ if (keymetadatalocation.equalsIgnoreCase("internal") && mode.equalsIgnoreCase("protectbulk")
+ && !showrevealkeybool) {
+ if (protectedData.length() > 7)
+ protectedData = protectedData.substring(7);
+ }
+ System.out.println(protectedData);
+ rownbranddata.add(dataIndex);
+ rownbranddata.add(new String(protectedData));
+ replies.add(rownbranddata);
+ rownbranddata = new JsonArray();
+ if (mode.equals("protectbulk")) {
+ if (keymetadatalocation.equalsIgnoreCase("external")
+ && mode.equalsIgnoreCase("protectbulk")) {
+ externalkeymetadata = protectedDataObject.get("external_version").getAsString();
+ System.out.println("Protected Data ext key metadata need to store this: "
+ + externalkeymetadata);
+
+ }
+ }
+ } else if (protectedDataObject.has("error_message")) {
+ String errorMessage = protectedDataObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ bqErrorMap.put(i, errorMessage);
+ bad_data = true;
+ } else
+ System.out.println("unexpected json value from results: ");
+ dataIndex++;
+
+ }
+
+ crdp_payload_array = new JsonArray();
+ protection_policy_buff = new StringBuffer();
+ numberofchunks++;
+ totalcount = totalcount + count;
+ count = 0;
+
+ } else {
+ System.err.println("Request failed with status code: " + crdp_response.code());
+ }
+ }
+ crdp_response.close();
+ System.out.println("total chuncks " + numberofchunks);
+
+ result.add("data", replies);
+
+ snowflakereturnstring = result.toString();
+
+ } catch (Exception e) {
+ System.out.println("in exception with " + e.getMessage());
+ snowflakereturnstring = "exception ";
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
+
+ result = new JsonObject();
+ replies = new JsonArray();
+ rownbranddata = new JsonArray();
+
+ for (int i = 0; i < snowflakedata.size(); i++) {
+
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ for (int j = 0; j < snowflakerow.size(); j++) {
+ if (j == 1) {
+ // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
+ // FPE example
+ String sensitive = checkValid(snowflakerow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null")
+ || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+
+ }
+ encdata = sensitive;
+
+ } else {
+ System.out.println("normal number data" + sensitive);
+ }
+ rownbranddata.add(sensitive);
+ replies.add(rownbranddata);
+ rownbranddata = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ int row_number = snowflakecolumn.getAsInt();
+ rownbranddata.add(row_number);
+ }
+ }
+ }
+
+ result.add("data", replies);
+ JsonObject inputJsonObject = new JsonObject();
+ String bodyString = result.toString();
+ inputJsonObject.addProperty("statusCode", 200);
+ inputJsonObject.addProperty("body", bodyString);
+
+ snowflakereturnstring = inputJsonObject.toString();
+ System.out.println(" new data " + snowflakereturnstring);
+
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+
+ } finally {
+
+ }
+ System.out.println(snowflakereturnstring);
+ // response.getWriter().write(snowflakereturnstring);
+
+ }
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer snowflakereturndatasb = new StringBuffer();
+
+ snowflakereturndatasb.append("{ \"statusCode\":");
+ snowflakereturndatasb.append(statusCode);
+ snowflakereturndatasb.append(",");
+ snowflakereturndatasb.append(" \"body\": {");
+ snowflakereturndatasb.append(" \"data\": [");
+ snowflakereturndatasb.append("] }}");
+ System.out.println("in exception with ");
+ return snowflakereturndatasb.toString();
+ }
+
+ public String checkValid(JsonArray snowrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (snowrow != null && snowrow.size() > 0) {
+ JsonElement element = snowrow.get(1);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+
+ boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
+
+ return founduserinuserset;
+
+ }
+
+ @Override
+ public void service(HttpRequest request, HttpResponse response) throws Exception {
+ // TODO Auto-generated method stub
+
+ }
+}
\ No newline at end of file
diff --git a/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCRDPFPETester.java b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCRDPFPETester.java
new file mode 100644
index 00000000..5dc1efa9
--- /dev/null
+++ b/database/snowflake/Thales-Snow-GCP-UDF/src/test/java/ThalesGCPSnowCRDPFPETester.java
@@ -0,0 +1,432 @@
+
+import com.google.cloud.functions.HttpFunction;
+import com.google.cloud.functions.HttpRequest;
+import com.google.cloud.functions.HttpResponse;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonPrimitive;
+
+import okhttp3.MediaType;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
+import java.math.BigInteger;
+import java.util.HashMap;
+import java.util.Map;
+
+import java.util.logging.Logger;
+
+/* This test app to test the logic for a Snowflake Database User Defined Function(UDF).
+ * It is an example of how to use Thales CipherTrust REST Application Dataprotection (CRDP)
+ * to protect sensitive data in a column. This example uses Format Preserve Encryption (FPE) to maintain the original format of the
+ * data so applications or business intelligence tools do not have to change in order to use these columns.
+*
+* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
+* for all possible data sizes and combinations of encryption algorithms and IV, etc.
+* Was tested with CM 2.14 and CRDP 1.0
+* For more information on CRDP see link below.
+https://thalesdocs.com/ctp/con/crdp/latest/index.html
+* For more information on Snowflake External Functions see link below.
+https://docs.snowflake.com/en/sql-reference/external-functions-creating-gcp
+ *
+ *@author mwarner
+ *
+ */
+public class ThalesGCPSnowCRDPFPETester implements HttpFunction {
+// @Override
+
+// @Override
+
+ private static final Gson gson = new Gson();
+ private static final String BADDATATAG = new String("9999999999999999");
+
+ private static final String REVEALRETURNTAG = new String("data");
+ private static final String PROTECTRETURNTAG = new String("protected_data");
+
+ public static void main(String[] args) throws Exception {
+
+ ThalesGCPSnowCRDPFPETester nw2 = new ThalesGCPSnowCRDPFPETester();
+
+ String protectnull = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"\" ],\r\n"
+ + " [1, \"somemoredatat\" ],\r\n" + " [ 2, \"null\" ]\r\n" + " ]\r\n" + "}";
+
+ String protect = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"pagetest\" ],\r\n"
+ + " [1, \"null\" ],\r\n" + " [2, \"0\" ],\r\n"
+ + " [ 3, \"life, the universe, and everything\" ]\r\n" + " ]\r\n" + "}";
+
+ String reveal_ext_alpha = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"wSf2fghX\" ],\r\n"
+ + " [1, \"xI\" ],\r\n" + " [2, \"notvalid0\" ],\r\n"
+ + " [ 3, \"Qqce, QBv d7ErVAPM, teq ZD6sjHPecT\" ]\r\n" + " ]\r\n" + "}";
+
+ String reveal_ext_nbr = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"730712653\" ],\r\n"
+ + " [1, \"4108-531\" ],\r\n" + " [2, \"0\" ],\r\n" + " [ 3, \"9999999999999999\" ]\r\n"
+ + " ]\r\n" + "}";
+
+ String protectnbr = "{ \"data\":\r\n" + " [\r\n" + " [ 0, \"345245667\" ],\r\n"
+ + " [1, \"4578-094\" ],\r\n" + " [2, \"0\" ],\r\n" + " [ 3, \"null\" ]\r\n" + " ]\r\n"
+ + "}";
+
+ // "data":[[0,"730712653"],[1,"4108-531"],[2,"0"],[3,"9999999999999999"]]}
+ String response = null;
+ nw2.service(protectnbr, response);
+
+ }
+
+ public void service(String request, String response) throws Exception {
+ Map snowErrorMap = new HashMap();
+ String encdata = "";
+ String snowflakereturnstring = null;
+ JsonArray snowflakedata = null;
+ int statusCode = 200;
+
+ String keyName = "testfaas";
+ String crdpip = System.getenv("CRDPIP");
+ String userName = System.getenv("CMUSER");
+ String password = System.getenv("CMPWD");
+
+ // returnciphertextforuserwithnokeyaccess = is a environment variable to express how data should be returned
+ // when the user above does not have access to the key and if doing a
+ // lookup in the userset and the user does not exist. If returnciphertextforuserwithnokeyaccess = no
+ // then an error will be returned to the query, else the results set will provide ciphertext.
+ String returnciphertextforuserwithnokeyaccess = System.getenv("returnciphertextforuserwithnokeyaccess");
+ // yes,no
+ boolean returnciphertextbool = returnciphertextforuserwithnokeyaccess.equalsIgnoreCase("yes");
+ // usersetlookup = should a userset lookup be done on the user from Cloud DB
+ // yes,no
+ String usersetlookup = System.getenv("usersetlookup");
+ // usersetidincm = should be the usersetid in CM to query.
+ String usersetID = System.getenv("usersetidincm");
+ // usersetlookupip = this is the IP address to query the userset. Currently it is the userset in CM but could be
+ // a memcache or other in memory db.
+ String userSetLookupIP = System.getenv("usersetlookupip");
+ boolean usersetlookupbool = usersetlookup.equalsIgnoreCase("yes");
+ String keymetadatalocation = System.getenv("keymetadatalocation");
+ String external_version_from_ext_source = System.getenv("keymetadata");
+ String protection_profile = System.getenv("protection_profile");
+ String mode = System.getenv("mode");
+ String datatype = System.getenv("datatype");
+ String dataKey = null;
+ String jsonTagForProtectReveal = null;
+
+ boolean bad_data = false;
+
+ String showrevealkey = "yes";
+
+ if (mode.equals("protect")) {
+ dataKey = "data";
+ jsonTagForProtectReveal = PROTECTRETURNTAG;
+ if (keymetadatalocation.equalsIgnoreCase("internal")) {
+ showrevealkey = System.getenv("showrevealinternalkey");
+ if (showrevealkey == null)
+ showrevealkey = "yes";
+ }
+ } else {
+ dataKey = "protected_data";
+ jsonTagForProtectReveal = REVEALRETURNTAG;
+ }
+ boolean showrevealkeybool = showrevealkey.equalsIgnoreCase("yes");
+
+ String snowflakeuser = "snowflakeuser";
+
+ JsonObject result = new JsonObject();
+ JsonArray replies = new JsonArray();
+ JsonArray rownbranddata = new JsonArray();
+
+ try {
+
+ // This code is only to be used when input data contains user info.
+ /*
+ * if (usersetlookupbool) { // make sure cmuser is in Application Data Protection Clients Group
+ *
+ * boolean founduserinuserset = findUserInUserSet(bigquerysessionUser, userName, password, usersetID,
+ * userSetLookupIP); // System.out.println("Found User " + founduserinuserset); if (!founduserinuserset)
+ * throw new CustomException("1001, User Not in User Set", 1001);
+ *
+ * } else { usersetlookupbool = false; }
+ */
+
+ try {
+ JsonElement requestParsed = gson.fromJson(request, JsonElement.class);
+ JsonObject requestJson = null;
+
+ if (requestParsed != null && requestParsed.isJsonObject()) {
+ requestJson = requestParsed.getAsJsonObject();
+ }
+
+ if (requestJson != null && requestJson.has("data")) {
+ snowflakedata = requestJson.getAsJsonArray("data");
+
+ }
+
+ } catch (JsonParseException e) {
+ System.out.println("Error parsing JSON: " + e.getMessage());
+
+ }
+
+ int row_number = 0;
+
+ String protectedData = null;
+ String externalkeymetadata = null;
+ String crdpjsonBody = null;
+ // String external_version_from_ext_source = "1004001";
+ // String external_version_from_ext_source = "1001001";
+
+ JsonObject crdp_payload = new JsonObject();
+
+ crdp_payload.addProperty("protection_policy_name", protection_profile);
+ String sensitive = null;
+ OkHttpClient client = new OkHttpClient().newBuilder().build();
+ MediaType mediaType = MediaType.parse("application/json");
+ String urlStr = "http://" + crdpip + ":8090/v1/" + mode;
+
+ // encrypt data
+ for (int i = 0; i < snowflakedata.size(); i++) {
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ for (int j = 0; j < snowflakerow.size(); j++) {
+
+ if (j == 1) {
+ // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
+ // FPE example
+ sensitive = checkValid(snowflakerow);
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null") || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+
+ }
+ encdata = sensitive;
+
+ } else {
+
+ crdp_payload.addProperty(dataKey, sensitive);
+ if (mode.equals("reveal")) {
+ crdp_payload.addProperty("username", snowflakeuser);
+ if (keymetadatalocation.equalsIgnoreCase("external")) {
+ crdp_payload.addProperty("external_version", external_version_from_ext_source);
+ }
+
+ }
+ crdpjsonBody = crdp_payload.toString();
+ System.out.println(crdpjsonBody);
+ RequestBody body = RequestBody.create(mediaType, crdpjsonBody);
+
+ Request crdp_request = new Request.Builder()
+ // .url("http://192.168.159.143:8090/v1/protect").method("POST", body)
+ .url(urlStr).method("POST", body).addHeader("Content-Type", "application/json")
+ .build();
+ Response crdp_response = client.newCall(crdp_request).execute();
+
+ if (crdp_response.isSuccessful()) {
+
+ // Parse JSON response
+ String responseBody = crdp_response.body().string();
+ Gson gson = new Gson();
+ JsonObject jsonObject = gson.fromJson(responseBody, JsonObject.class);
+
+ if (jsonObject.has(jsonTagForProtectReveal)) {
+ protectedData = jsonObject.get(jsonTagForProtectReveal).getAsString();
+ if (keymetadatalocation.equalsIgnoreCase("external")
+ && mode.equalsIgnoreCase("protect")) {
+ externalkeymetadata = jsonObject.get("external_version").getAsString();
+ System.out.println("Protected Data ext key metadata need to store this: "
+ + externalkeymetadata);
+ }
+
+ if (keymetadatalocation.equalsIgnoreCase("internal")
+ && mode.equalsIgnoreCase("protect") && !showrevealkeybool) {
+ if (protectedData.length() > 7)
+ protectedData = protectedData.substring(7);
+ }
+
+ } else if (jsonObject.has("error_message")) {
+ String errorMessage = jsonObject.get("error_message").getAsString();
+ System.out.println("error_message: " + errorMessage);
+ snowErrorMap.put(i, errorMessage);
+ bad_data = true;
+ } else
+ System.out.println("unexpected json value from results: ");
+
+ System.out.println("Protected Data: " + protectedData);
+
+ } else {
+ System.err.println("Request failed with status code: " + crdp_response.code());
+ }
+
+ crdp_response.close();
+
+ encdata = protectedData;
+
+ }
+
+ rownbranddata.add(encdata);
+ replies.add(rownbranddata);
+ rownbranddata = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ row_number = snowflakecolumn.getAsInt();
+ rownbranddata.add(row_number);
+
+ }
+
+ }
+
+ }
+
+ result.add("data", replies);
+ snowflakereturnstring = result.toString();
+
+ } catch (Exception e) {
+
+ System.out.println("in exception with " + e.getMessage());
+ snowflakereturnstring = "exception ";
+ if (returnciphertextbool) {
+ if (e.getMessage().contains("1401")
+ || (e.getMessage().contains("1001") || (e.getMessage().contains("1002")))) {
+
+ result = new JsonObject();
+ replies = new JsonArray();
+ rownbranddata = new JsonArray();
+
+ for (int i = 0; i < snowflakedata.size(); i++) {
+
+ JsonArray snowflakerow = snowflakedata.get(i).getAsJsonArray();
+
+ for (int j = 0; j < snowflakerow.size(); j++) {
+ if (j == 1) {
+ // String sensitive = snowflakecolumn.getAsJsonPrimitive().toString();
+ // FPE example
+ String sensitive = checkValid(snowflakerow);
+
+ if (sensitive.contains("notvalid") || sensitive.equalsIgnoreCase("null")) {
+ if (datatype.equalsIgnoreCase("charint") || datatype.equalsIgnoreCase("nbr")) {
+ if (sensitive.contains("notvalid")) {
+ System.out.println("adding null not charint or nbr");
+ sensitive = sensitive.replace("notvalid", "");
+ } else
+ sensitive = BADDATATAG;
+
+ } else if (sensitive.equalsIgnoreCase("null")
+ || sensitive.equalsIgnoreCase("notvalid")) {
+
+ } else if (sensitive.contains("notvalid")) {
+ sensitive = sensitive.replace("notvalid", "");
+
+ }
+ encdata = sensitive;
+
+ } else {
+ System.out.println("normal number data" + sensitive);
+ }
+ rownbranddata.add(sensitive);
+ replies.add(rownbranddata);
+ rownbranddata = new JsonArray();
+
+ } else {
+ JsonPrimitive snowflakecolumn = snowflakerow.get(j).getAsJsonPrimitive();
+ int row_number = snowflakecolumn.getAsInt();
+ rownbranddata.add(row_number);
+ }
+ }
+ }
+
+ result.add("data", replies);
+ JsonObject inputJsonObject = new JsonObject();
+ String bodyString = result.toString();
+ inputJsonObject.addProperty("statusCode", 200);
+ inputJsonObject.addProperty("body", bodyString);
+
+ snowflakereturnstring = inputJsonObject.toString();
+ System.out.println(" new data " + snowflakereturnstring);
+
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+ } else {
+ statusCode = 400;
+ snowflakereturnstring = formatReturnValue(statusCode);
+ e.printStackTrace(System.out);
+ }
+
+ } finally {
+
+ }
+ System.out.println(snowflakereturnstring);
+ // response.getWriter().write(snowflakereturnstring);
+
+ }
+
+ public String formatReturnValue(int statusCode)
+
+ {
+ StringBuffer snowflakereturndatasb = new StringBuffer();
+
+ snowflakereturndatasb.append("{ \"statusCode\":");
+ snowflakereturndatasb.append(statusCode);
+ snowflakereturndatasb.append(",");
+ snowflakereturndatasb.append(" \"body\": {");
+ snowflakereturndatasb.append(" \"data\": [");
+ snowflakereturndatasb.append("] }}");
+ System.out.println("in exception with ");
+ return snowflakereturndatasb.toString();
+ }
+
+ public String checkValid(JsonArray snowrow) {
+ String inputdata = null;
+ String notvalid = "notvalid";
+ if (snowrow != null && snowrow.size() > 0) {
+ JsonElement element = snowrow.get(1);
+ if (element != null && !element.isJsonNull()) {
+ inputdata = element.getAsString();
+ if (inputdata.isEmpty() || inputdata.length() < 2) {
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("Sensitive data is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+ } else {
+ // System.out.println("bigquerytrow is null or empty.");
+ inputdata = notvalid + inputdata;
+ }
+
+ return inputdata;
+
+ }
+
+ public boolean findUserInUserSet(String userName, String cmuserid, String cmpwd, String userSetID,
+ String userSetLookupIP) throws Exception {
+
+ CMUserSetHelper cmuserset = new CMUserSetHelper(userSetID, userSetLookupIP);
+
+ String jwthtoken = CMUserSetHelper.geAuthToken(cmuserset.authUrl, cmuserid, cmpwd);
+ String newtoken = "Bearer " + CMUserSetHelper.removeQuotes(jwthtoken);
+
+ boolean founduserinuserset = cmuserset.findUserInUserSet(userName, newtoken);
+
+ return founduserinuserset;
+
+ }
+
+ @Override
+ public void service(HttpRequest request, HttpResponse response) throws Exception {
+ // TODO Auto-generated method stub
+
+ }
+}
\ No newline at end of file
diff --git a/database/snowflake/documentation/Snowflake_with_CADP.pdf b/database/snowflake/documentation/Snowflake_with_CADP.pdf
new file mode 100644
index 00000000..91c910f1
Binary files /dev/null and b/database/snowflake/documentation/Snowflake_with_CADP.pdf differ
diff --git a/database/snowflake/documentation/Snowflake_with_CRDP.pdf b/database/snowflake/documentation/Snowflake_with_CRDP.pdf
new file mode 100644
index 00000000..b76617a4
Binary files /dev/null and b/database/snowflake/documentation/Snowflake_with_CRDP.pdf differ
diff --git a/database/vertica/src/main/java/com/vertica/JavaLibs/ThalesProtectAppDecryptFPE.java b/database/vertica/src/main/java/com/vertica/JavaLibs/ThalesProtectAppDecryptFPE.java
index 9c2767a2..03e4cd86 100644
--- a/database/vertica/src/main/java/com/vertica/JavaLibs/ThalesProtectAppDecryptFPE.java
+++ b/database/vertica/src/main/java/com/vertica/JavaLibs/ThalesProtectAppDecryptFPE.java
@@ -36,15 +36,9 @@
public class ThalesProtectAppDecryptFPE extends ScalarFunctionFactory {
String username = "admin";
- String password = "YoursuperSecret!";
String keyName = "MyAESEncryptionKey26";
- int authTagLength = Integer.parseInt("128");
- String iv = "6162636465666768696a6b6c";
- String aad = "6162636465666768696a6b6c";
NAESession session = null;
NAEKey key = null;
- byte[] ivBytes = IngrianProvider.hex2ByteArray(iv);
- byte[] aadBytes = IngrianProvider.hex2ByteArray(aad);
String tweakData = null;
String tweakAlgo = null;
FPEParameterAndFormatSpec param = null;
@@ -76,7 +70,7 @@ public class ThalesDecryptFPEData extends ScalarFunction {
public void setup(ServerInterface srvInterface, SizedColumnTypes argTypes) {
srvInterface.log("In setup");
- session = NAESession.getSession(username, password.toCharArray(), "hello".toCharArray());
+ session = NAESession.getSession(username, "Yoursuper!".toCharArray(), "hello".toCharArray());
key = NAEKey.getSecretKey(keyName, session);
param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
.build();
diff --git a/database/vertica/src/main/java/com/vertica/JavaLibs/ThalesProtectAppEncryptFPE.java b/database/vertica/src/main/java/com/vertica/JavaLibs/ThalesProtectAppEncryptFPE.java
index dcafd4b7..e3196883 100644
--- a/database/vertica/src/main/java/com/vertica/JavaLibs/ThalesProtectAppEncryptFPE.java
+++ b/database/vertica/src/main/java/com/vertica/JavaLibs/ThalesProtectAppEncryptFPE.java
@@ -41,15 +41,9 @@
public class ThalesProtectAppEncryptFPE extends ScalarFunctionFactory {
String username = "admin";
- String password = "YoursuperSecret!";
String keyName = "MyAESEncryptionKey26";
- int authTagLength = Integer.parseInt("128");
- String iv = "6162636465666768696a6b6c";
- String aad = "6162636465666768696a6b6c";
NAESession session = null;
NAEKey key = null;
- byte[] ivBytes = IngrianProvider.hex2ByteArray(iv);
- byte[] aadBytes = IngrianProvider.hex2ByteArray(aad);
String tweakData = null;
String tweakAlgo = null;
FPEParameterAndFormatSpec param = null;
@@ -81,7 +75,7 @@ public class ThalesEncryptFPEData extends ScalarFunction {
public void setup(ServerInterface srvInterface, SizedColumnTypes argTypes) {
srvInterface.log("In setup");
- session = NAESession.getSession(username, password.toCharArray(), "hello".toCharArray());
+ session = NAESession.getSession(username, "Yoursuper!".toCharArray(), "hello".toCharArray());
key = NAEKey.getSecretKey(keyName, session);
param = new FPEParameterAndFormatBuilder(tweakData).set_tweakAlgorithm(tweakAlgo)
.build();
diff --git a/demos/crypto-tool-cm/package.json b/demos/crypto-tool-cm/package.json
index 6d5de2ae..c4fd2cc2 100644
--- a/demos/crypto-tool-cm/package.json
+++ b/demos/crypto-tool-cm/package.json
@@ -18,7 +18,7 @@
"axios": "^1.2.1",
"eslint": "8.29.0",
"eslint-config-next": "13.0.6",
- "next": "13.5.0",
+ "next": "14.2.10",
"react": "18.2.0",
"react-dom": "18.2.0",
"react-toastify": "^9.1.1",
diff --git a/key_management/csharp/NaeGetKeyVersionById.cs b/key_management/csharp/NaeGetKeyVersionById.cs
new file mode 100644
index 00000000..b3aba5c7
--- /dev/null
+++ b/key_management/csharp/NaeGetKeyVersionById.cs
@@ -0,0 +1,102 @@
+using CADP.NetCore.KeyManagement;
+using CADP.NetCore.Sessions;
+
+namespace CADP.NetCoreNaeSamples
+{
+ /// ***************Prerequisites**************************
+ /// NAE versioned key should be present on CipherTrust Manager and UUID must be known.
+ /// ******************************************************
+ ///
+ ///
+ /// This sample shows how to fetch key name, key version and correspondig version header bytes using UUID for NAE versioned key.
+ /// For non versioned key, -1 will be returned with Null header bytes.
+ ///
+ class NaeGetKeyVersionById
+ {
+ static void Main(string[] args)
+ {
+ NaeSession session;
+ NaeKeyManagement nkm;
+
+
+ /*Read Username and password*/
+ Console.Write("Enter username: ");
+ string user = Console.ReadLine();
+ Console.Write("Enter password: ");
+ string pass = string.Empty;
+ ConsoleKeyInfo consoleKeyInfo;
+
+ do
+ {
+ consoleKeyInfo = Console.ReadKey(true);
+
+ // Handle backspace and remove the key.
+ if (consoleKeyInfo.Key == ConsoleKey.Backspace)
+ {
+ Console.Write("\b \b");
+ pass = (pass.Length > 0) ? pass.Remove(pass.Length - 1, 1) : pass;
+ }
+ else
+ {
+ // Not adding the function keys, other keys having key char as '\0' in the password string.
+ if (consoleKeyInfo.KeyChar != '\0')
+ {
+ pass += consoleKeyInfo.KeyChar;
+ Console.Write("*");
+ }
+ }
+ }
+ // Stops Receving Keys Once Enter is Pressed
+ while (consoleKeyInfo.Key != ConsoleKey.Enter);
+
+ // cleaning up the newline character
+ pass = pass.Replace("\r", "");
+ Console.WriteLine();
+
+ try
+ {
+ /*Read the CADP.NETCore_Properties.xml from the nuget folder.
+ In case, of multiple versions available it will take the latest one.
+ Please update the code in case of below requirement:
+ 1. latest version is not required to be picked.
+ 2. custom location for the file
+ */
+
+ var propertyFilePath = string.Empty;
+ string path = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
+ var cadpPackage = Path.Combine(path, ".nuget", "packages", "ciphertrust.cadp.netcore");
+ var highestPackage = Directory.GetDirectories(cadpPackage).Select(x => Path.GetFileName(x)).OrderBy(x => Path.GetFileName(x)).Last();
+ propertyFilePath = Path.Combine(cadpPackage, highestPackage, "content", "CADP.NETCore_Properties.xml");
+
+ /* Create a new NAE Session using the username and password */
+ session = new NaeSession(user, pass, propertyFilePath);
+ Console.WriteLine("NaeSession created successfully.");
+
+ Console.WriteLine("Enter the UUID");
+ string uuid = Console.ReadLine();
+
+ nkm = new NaeKeyManagement(session);
+
+ VersionInfo keyDetails = nkm.GetKeyVersionById(uuid, NaeKeyManagement.KeyIdType.UUID);
+
+ Console.WriteLine("Keyname: " + keyDetails.KeyName);
+
+ //For non versioned NAE key, returned version will be -1 and header bytes are null
+ if (keyDetails.Version != -1)
+ {
+ Console.WriteLine($"Version: {keyDetails.Version}");
+ // For printing to console converting the header bytes to Base64 encoded string
+ Console.WriteLine($"Version Header (B64 encoded): {Convert.ToBase64String(keyDetails.Header)}");
+ }
+ else
+ {
+ Console.WriteLine("Founded Key is non versioned");
+ }
+ }
+ catch (Exception ex)
+ {
+ Console.WriteLine($"Error in running the code. {ex.Message}");
+ }
+ }
+ }
+}
diff --git a/key_management/csharp/README.md b/key_management/csharp/README.md
index 4c20e630..f4ceb742 100644
--- a/key_management/csharp/README.md
+++ b/key_management/csharp/README.md
@@ -3,6 +3,8 @@
## Overview
The following samples shows how to manage keys.
+* NaeGetKeyVersionById.cs
+ * This sample shows how to *fetch key name, key version and corresponding version header bytes using UUID for NAE versioned key*.
* NaeKeyManagement.cs
* This sample shows how to *get the attributes of the key*.
* ExportWrappedKey.cs
@@ -12,6 +14,7 @@ The following samples shows how to manage keys.
In order to run C# samples,
1. .NET 6.0 or higher must be installed.
1. CipherTrust.CADP.NETCore NuGet package must be installed.
+1. Key must be present on CipherTrust Manager. For NaeGetKeyVersionById.cs, key should be NAE versioned.
## Usage:
1. Create a console application. Let's say `SampleApp`.
diff --git a/key_management/java/KeyNameSample.java b/key_management/java/KeyNameSample.java
index ec60b045..380ab998 100644
--- a/key_management/java/KeyNameSample.java
+++ b/key_management/java/KeyNameSample.java
@@ -64,7 +64,11 @@ else if ("-max".equals(args[i]))
CustomAttributes attr = new CustomAttributes();
if(attributeValue != null){
attr.addAttributeForKeyName(attributeName, attributeValue);
- attr.addAttributeForKeyName(attributeName + "-1", attributeValue);
+ /*
+ * Not supported before CM 2.17
+ * attr.addAttributeForKeyName(attributeName + "-1", attributeValue);
+ */
+
}
UserKeysDetail keyNames = NAEKey.getKeyNames(attr, fingerprint, offset, max, session, ConjunctiveOperator.OR);
System.out.println("Key count: " + keyNames.getKeyCount());
diff --git a/miscellaneous/csharp/AddTransactionIdSample.cs b/miscellaneous/csharp/AddTransactionIdSample.cs
index 4b4e8890..e150f32e 100644
--- a/miscellaneous/csharp/AddTransactionIdSample.cs
+++ b/miscellaneous/csharp/AddTransactionIdSample.cs
@@ -104,8 +104,10 @@ 1. latest version is not required to be picked.
//The nonce sizes supported by this instance: 12 bytes (96 bits).
//which should be a unique value for every operation with the same key.
- byte[] nonce = { 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x30, 0x31 };
-
+ byte[] nonce = new byte[12];
+ Random random = new Random();
+ random.NextBytes(nonce);
+
try
{
byte[] tag = null;
diff --git a/miscellaneous/csharp/README.md b/miscellaneous/csharp/README.md
index 0eee68ff..9ce5ee6f 100644
--- a/miscellaneous/csharp/README.md
+++ b/miscellaneous/csharp/README.md
@@ -3,6 +3,8 @@
## Overview
PassPhraseSecureUtility.cs sample shows how to use PassPhraseEncryption method provided in CADP.NetCore.Utility namespace. This utility can be used to obfuscate the login credentials as well as client certificate passphrase.
+TestCryptoDataUtility.cs sample shows how to use CryptoDataUtility.dll for user to perform crypto operations without specifying the key name. To accomplish this, the ciphertext, at the time of encryption, is bundled with the same meta data, of the key, which was used for encryption.
+
AddTransactionIdSample.cs sample shows how to set and unset Transaction ID in the log entries using TransactionID property of LoggerWrapper class.
## Prerequisites:
@@ -13,9 +15,9 @@ In order to run C# samples,
## Usage:
1. Create a console application. Let's say `SampleApp`.
1. From the available sample cs files, add anyone to the project.
-1. Add the CipherTrust.CADP.NETCore NuGet package to the project and build using command `dotnet build -c Release`. To know more about NuGet package and how to add it, refer to [CipherTrust.CADP.NETCore](https://www.nuget.org/packages/CipherTrust.CADP.NETCore/).
+1. Add the CipherTrust.CADP.NETCore NuGet package to the project. To know more about NuGet package and how to add it, refer to [CipherTrust.CADP.NETCore](https://www.nuget.org/packages/CipherTrust.CADP.NETCore/)
+1. [**For TestCryptoDataUtility sample only**] Add CryptoDataUtility.dll as reference in project. On installing CADP for .NetCore Nuget package, this dll would be available at "%UserProfile%\\.nuget\packages\ciphertrust.cadp.netcore\\\utility\" folder.
1. Build using command `dotnet build -c Release`.
-
1. Use either of the following commands to run-
* `dotnet run` command to run the project at a terminal prompt.
* `dotnet` command to run `SampleApp.dll` at location `\bin\release\net6.0` using terminal. Example: `dotnet SampleApp.dll`
@@ -23,4 +25,4 @@ In order to run C# samples,
* Provide the command line parameter as required for both the samples.
## More Information
-For more information on CADP for .NET Core, refer to the [CADP for .NET Core user guide](https://thalesdocs.com/ctp/con/cadp/cadp-netcore/latest/index.html).
+For more information on CADP for .NET Core, refer to the [CADP for .NET Core user guide](https://thalesdocs.com/ctp/con/cadp/cadp-netcore/alpha-8.14.0/index.html).
diff --git a/miscellaneous/csharp/TestCryptoDataUtility.cs b/miscellaneous/csharp/TestCryptoDataUtility.cs
new file mode 100644
index 00000000..ead49921
--- /dev/null
+++ b/miscellaneous/csharp/TestCryptoDataUtility.cs
@@ -0,0 +1,101 @@
+using CADP.NetCore.Sessions;
+using CryptoDataUtility;
+using System;
+using System.IO;
+using System.Linq;
+
+namespace CADP.NetCoreNaeSamples
+{
+ class TestCryptoDataUtility
+ {
+ static void Main(string[] args)
+ {
+ var errorMessage = @" Please provide command line arguments to use the sample!
+ Usage
+ Parameter 1 : Key name to be used for crypto operation (already generated)
+ Parameter 2 : Text to encrypt.";
+
+ if(args == null || args.Length != 2)
+ {
+ Console.WriteLine(errorMessage);
+ return;
+ }
+
+ try
+ {
+ /*Read the CADP.NETCore_Properties.xml from the nuget folder.
+ In case, of multiple versions available it will take the latest one.
+ Please update the code in case of below requirement:
+ 1. latest version is not required to be picked.
+ 2. custom location for the file
+ */
+ var propertyFilePath = string.Empty;
+ string path = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
+ var cadpPackage = Path.Combine(path, ".nuget", "packages", "ciphertrust.cadp.netcore");
+ var highestPackage = Directory.GetDirectories(cadpPackage).Select(x => Path.GetFileName(x)).OrderBy(x => Path.GetFileName(x)).Last();
+ propertyFilePath = Path.Combine(cadpPackage, highestPackage, "content", "CADP.NETCore_Properties.xml");
+
+ // Reading the command line arguments.
+ string keyName = args[0];
+ string plaintext = args[1];
+
+ /*Read Username and password*/
+ Console.Write("Enter username: ");
+ string user = Console.ReadLine();
+ Console.Write("Enter password: ");
+ string pass = string.Empty;
+ ConsoleKeyInfo consoleKeyInfo;
+
+ do
+ {
+ consoleKeyInfo = Console.ReadKey(true);
+
+ // Handle backspace and remove the key.
+ if (consoleKeyInfo.Key == ConsoleKey.Backspace)
+ {
+ Console.Write("\b \b");
+ pass = (pass.Length > 0) ? pass.Remove(pass.Length - 1, 1) : pass;
+ }
+ else
+ {
+ // Not adding the function keys, other keys having key char as '\0' in the password string.
+ if (consoleKeyInfo.KeyChar != '\0')
+ {
+ pass += consoleKeyInfo.KeyChar;
+ Console.Write("*");
+ }
+ }
+ }
+ // Stops Receving Keys Once Enter is Pressed
+ while (consoleKeyInfo.Key != ConsoleKey.Enter);
+
+ // cleaning up the newline character
+ pass = pass.Replace("\r", "");
+ Console.WriteLine();
+
+ // construct NAESession object
+ NaeSession session = new NaeSession(user, pass, propertyFilePath);
+
+ // or change to a different NAESession constructor
+ // construct encryption utility object
+ SymmetricEncryptionUtility utility = new SymmetricEncryptionUtility(session);
+
+ // Encrypt
+ byte[] encrypted = utility.Encrypt(System.Text.Encoding.UTF8.GetBytes(plaintext),
+ keyName);
+ Console.WriteLine("Encrypted: " + Convert.ToBase64String(encrypted));
+
+ // Decrypt
+ //Note :- We are not giving any keyname here for decryption, only
+ //thing provided is the cipher text.
+ byte[] decrypted = utility.Decrypt(encrypted);
+ Console.WriteLine("Decrypted: " + System.Text.Encoding.UTF8.GetString(decrypted));
+ Console.WriteLine("Done!");
+ }
+ catch(Exception ex)
+ {
+ Console.WriteLine($"Error Occurred: {ex.Message}");
+ }
+ }
+ }
+}
diff --git a/rest/.classpath b/rest/.classpath
deleted file mode 100644
index 75b806fd..00000000
--- a/rest/.classpath
+++ /dev/null
@@ -1,44 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/rest/.project b/rest/.project
deleted file mode 100644
index 6a403972..00000000
--- a/rest/.project
+++ /dev/null
@@ -1,34 +0,0 @@
-
-
- cmhelper
-
-
-
-
-
- org.eclipse.jdt.core.javabuilder
-
-
-
-
- org.eclipse.m2e.core.maven2Builder
-
-
-
-
-
- org.eclipse.jdt.core.javanature
- org.eclipse.m2e.core.maven2Nature
-
-
-
- 1626690847887
-
- 30
-
- org.eclipse.core.resources.regexFilterMatcher
- node_modules|.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__
-
-
-
-
diff --git a/rest/.settings/org.eclipse.jdt.apt.core.prefs b/rest/.settings/org.eclipse.jdt.apt.core.prefs
deleted file mode 100644
index d4313d4b..00000000
--- a/rest/.settings/org.eclipse.jdt.apt.core.prefs
+++ /dev/null
@@ -1,2 +0,0 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.apt.aptEnabled=false
diff --git a/rest/.settings/org.eclipse.jdt.core.prefs b/rest/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index ac8e7500..00000000
--- a/rest/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5
-org.eclipse.jdt.core.compiler.compliance=1.5
-org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
-org.eclipse.jdt.core.compiler.processAnnotations=disabled
-org.eclipse.jdt.core.compiler.release=disabled
-org.eclipse.jdt.core.compiler.source=1.5
diff --git a/rest/.settings/org.eclipse.m2e.core.prefs b/rest/.settings/org.eclipse.m2e.core.prefs
deleted file mode 100644
index f897a7f1..00000000
--- a/rest/.settings/org.eclipse.m2e.core.prefs
+++ /dev/null
@@ -1,4 +0,0 @@
-activeProfiles=
-eclipse.preferences.version=1
-resolveWorkspaceProjects=true
-version=1
diff --git a/rest/README.md b/rest/README.md
index 078bc4b4..4e02eb0f 100644
--- a/rest/README.md
+++ b/rest/README.md
@@ -2,14 +2,9 @@
Integrations or Sample Code for applications using REST calls to our web services products:
-* CipherTrust Application Data Protection (CADP) Web Services
- * ProtectApp Web Services (legacy)
- * Vormetric Application Encryption (VAE) Web Services (legacy)
-* CipherTrust Vaulted Tokenization (CT-V)
- * ProtectApp (PA) Web Services (legacy)
+* CipherTrust REST Data Protection (CRDP)
* CipherTrust Vaultless Tokenization (CT-VL)
- * SafeNet Vaultless Tokenization (SVT) (legacy)
- * Vormetric Tokenization Server (VTS) (legacy)
- * CipherTrust Manager REST API
+* CipherTrust Application Data Protection (CADP) Web Services
+* CipherTrust Manager REST API
## Sample Code
diff --git a/rest/documentation/using-ctm-protectapp-ctm-rest-aws-paas.pdf b/rest/cmrest/documentation/using-ctm-protectapp-ctm-rest-aws-paas.pdf
similarity index 100%
rename from rest/documentation/using-ctm-protectapp-ctm-rest-aws-paas.pdf
rename to rest/cmrest/documentation/using-ctm-protectapp-ctm-rest-aws-paas.pdf
diff --git a/rest/pom.xml b/rest/cmrest/pom.xml
similarity index 97%
rename from rest/pom.xml
rename to rest/cmrest/pom.xml
index 58c30d5b..7e76e6be 100644
--- a/rest/pom.xml
+++ b/rest/cmrest/pom.xml
@@ -1,24 +1,24 @@
-
- 4.0.0
-
- com.thales.cm.rest
- cmhelper
- 0.0.1-SNAPSHOT
- jar
-
- cmhelper
- http://maven.apache.org
-
-
- com.squareup.okhttp3
- mockwebserver
- 4.7.2
-
-
- com.jayway.jsonpath
- json-path
- 2.4.0
-
-
-
+
+ 4.0.0
+
+ com.thales.cm.rest
+ cmhelper
+ 0.0.1-SNAPSHOT
+ jar
+
+ cmhelper
+ http://maven.apache.org
+
+
+ com.squareup.okhttp3
+ mockwebserver
+ 4.7.2
+
+
+ com.jayway.jsonpath
+ json-path
+ 2.4.0
+
+
+
diff --git a/rest/src/main/java/com/thales/cm/rest/cmhelper/App.java b/rest/cmrest/src/main/java/com/thales/cm/rest/cmhelper/App.java
similarity index 96%
rename from rest/src/main/java/com/thales/cm/rest/cmhelper/App.java
rename to rest/cmrest/src/main/java/com/thales/cm/rest/cmhelper/App.java
index 09b523e6..a4bb9e72 100644
--- a/rest/src/main/java/com/thales/cm/rest/cmhelper/App.java
+++ b/rest/cmrest/src/main/java/com/thales/cm/rest/cmhelper/App.java
@@ -1,54 +1,54 @@
-package com.thales.cm.rest.cmhelper;
-
-import java.io.IOException;
-
-/**
- * Hello world!
- *
- */
-public class App
-{
- public static void main( String[] args )
- {
-
- CipherTrustManagerHelper awsresrest = new CipherTrustManagerHelper();
-
- String results = null;
- String sensitive = "HelloWorld";
-
- awsresrest.username = "admin";
- awsresrest.password = "yourpwd";
- awsresrest.cmipaddress = "ipaddress";
-
- try {
- String tkn = awsresrest.getToken();
-
- awsresrest.key = "MyAESEncryptionKey26";
- System.out.println( "Original Data " + sensitive );
- results = awsresrest.cmRESTProtect( "gcm", sensitive, "encrypt");
- System.out.println( "Print results for encrypt" + results);
- results = awsresrest.cmRESTProtect( "gcm", results, "decrypt");
- System.out.println( "Print results for decrypt " + results);
-
- awsresrest.key = "rsa-key5";
- results = awsresrest.cmRESTSign( "SHA1", "na", sensitive,"sign");
- System.out.println( "Print results for sign" + results);
- results = awsresrest.cmRESTSign( "SHA1", results, sensitive, "signv");
- System.out.println( "Print results for verify " + results);
-
- awsresrest.key = "hmacsha256-1";
- results = awsresrest.cmRESTMac( "na", sensitive,"mac");
- System.out.println( "Print results for mac " + results);
- results = awsresrest.cmRESTMac( results, sensitive, "macv");
- System.out.println( "Print results for verify " + results);
-
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- } catch (Exception e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
-
- }
-}
+package com.thales.cm.rest.cmhelper;
+
+import java.io.IOException;
+
+/**
+ * Hello world!
+ *
+ */
+public class App
+{
+ public static void main( String[] args )
+ {
+
+ CipherTrustManagerHelper awsresrest = new CipherTrustManagerHelper();
+
+ String results = null;
+ String sensitive = "HelloWorld";
+
+ awsresrest.username = "admin";
+ awsresrest.password = "yourpwd";
+ awsresrest.cmipaddress = "ipaddress";
+
+ try {
+ String tkn = awsresrest.getToken();
+
+ awsresrest.key = "MyAESEncryptionKey26";
+ System.out.println( "Original Data " + sensitive );
+ results = awsresrest.cmRESTProtect( "gcm", sensitive, "encrypt");
+ System.out.println( "Print results for encrypt" + results);
+ results = awsresrest.cmRESTProtect( "gcm", results, "decrypt");
+ System.out.println( "Print results for decrypt " + results);
+
+ awsresrest.key = "rsa-key5";
+ results = awsresrest.cmRESTSign( "SHA1", "na", sensitive,"sign");
+ System.out.println( "Print results for sign" + results);
+ results = awsresrest.cmRESTSign( "SHA1", results, sensitive, "signv");
+ System.out.println( "Print results for verify " + results);
+
+ awsresrest.key = "hmacsha256-1";
+ results = awsresrest.cmRESTMac( "na", sensitive,"mac");
+ System.out.println( "Print results for mac " + results);
+ results = awsresrest.cmRESTMac( results, sensitive, "macv");
+ System.out.println( "Print results for verify " + results);
+
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
+ }
+}
diff --git a/rest/src/main/java/com/thales/cm/rest/cmhelper/CipherTrustManagerHelper.java b/rest/cmrest/src/main/java/com/thales/cm/rest/cmhelper/CipherTrustManagerHelper.java
similarity index 96%
rename from rest/src/main/java/com/thales/cm/rest/cmhelper/CipherTrustManagerHelper.java
rename to rest/cmrest/src/main/java/com/thales/cm/rest/cmhelper/CipherTrustManagerHelper.java
index 83815ac3..d893cb77 100644
--- a/rest/src/main/java/com/thales/cm/rest/cmhelper/CipherTrustManagerHelper.java
+++ b/rest/cmrest/src/main/java/com/thales/cm/rest/cmhelper/CipherTrustManagerHelper.java
@@ -1,1201 +1,1205 @@
-package com.thales.cm.rest.cmhelper;
-
-import java.io.IOException;
-import java.security.KeyManagementException;
-import java.security.KeyStore;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.security.cert.CertificateException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Base64;
-import java.util.Calendar;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.HttpsURLConnection;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLSession;
-import javax.net.ssl.SSLSocketFactory;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.TrustManagerFactory;
-import javax.net.ssl.X509TrustManager;
-
-import com.jayway.jsonpath.JsonPath;
-
-import okhttp3.CipherSuite;
-import okhttp3.ConnectionSpec;
-import okhttp3.MediaType;
-import okhttp3.OkHttpClient;
-import okhttp3.Request;
-import okhttp3.RequestBody;
-import okhttp3.Response;
-import okhttp3.TlsVersion;
-
-/* This example helper class simplifies the usage of REST calls to the Thales CipherTrust Manager by
- * exposing a few API's that shield some of the complexities of the json formating.
- * This examples also shows how to work with refresh token and use retry logic when jwt expires.
- * JWT duration = 300 seconds
-* Handles
-* 1.) encrypt/decrypt for gcm,rsa,fpe. (Note: RSA Decrypt returns values in base64 format)
-* 2.) mac/macv
-* 3.) sign/signv
-*
-* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
-* for all possible data sizes and combinations of encryption algorithms and IV, etc.
-* Was tested with CM 2.1 & 2.2
-* Uses the okhttp3 4.7.2 library.
-*
- */
-public class CipherTrustManagerHelper {
- String cmdebug = "0";
-
- public String token = null;
- public String key = null;
- public String cmipaddress;
- public String refreshtoken;
- public String keystorepwd;
- public String username;
- public String password;
- public String dataformat;
-
- public static final String endbracket = "}";
- public static final int digitblocklen = 56;
- public static final int alphablocklen = 32;
- public static final StringBuffer numberPattern = new StringBuffer(
- "01234567890123456789012345678901024567896743678905435678");
- public static final StringBuffer stringPattern = new StringBuffer("asdfghjklzxcvbnmqwertyuioplkjhgf");
- public static final StringBuffer combinedPattern = new StringBuffer("abcdefghijklmnopqrstuvwxyz012345");
- public static final String quote = "\"";
- public static final String comma = ",";
- public static final int wait = 60000;
- public static final int encoding_parameters_length = 11;
- public static final int version = 0;
- public static final String versiontag = "\"version\":";
- public static final String plaintexttag = "{\"plaintext\":";
- public static final String tag = "kBr5A0fbPjPg7lS1bB6wfw==";
- public static final String iv = "VCC3VwxWu6Z6jfQw";
- public static final String aadtag = "\"aad\":";
- public static final String padtag = "\"pad\":";
- public static final String rsapad = "\"oaep\"";
- public static final String idtag = "\"id\":";
- public static final String typetag = "\"type\":";
- public static final String name = "\"name\"";
- public static final String aad = "YXV0aGVudGljYXRl";
- public static final String ciphertexttag = "{\"ciphertext\":";
- public static final String tagtag = "\"tag\":";
- public static final String ivtag = "\"iv\":";
- public static final String modetag = "\"mode\":";
- public static final String filetagname = "etag:";
- public static final String filetagsep = "!";
- private static final String[] VALID_HOSTS = { "ipaddress1", "ipaddress2", "fqdn1" };
- public static final MediaType JSONOCTET = MediaType.get("application/octet-stream");
- public static final MediaType JSON = MediaType.get("application/json; charset=utf-8");
- public static final MediaType JSONTXT = MediaType.get("text/plain");
-
- OkHttpClient client = getUnsafeOkHttpClient();
-
- public CipherTrustManagerHelper() {
- super();
- Map env = System.getenv();
- // System.out.println("name of logger" + log.getName());
- // log.debug("this is a test");
- // log.info("this is a test");
- for (String envName : env.keySet()) {
- if (envName.equalsIgnoreCase("cmuserid")) {
- username = env.get(envName);
- if (cmdebug.equalsIgnoreCase("1"))
- System.out.println("cmuserid=" + username);
- } else if (envName.equalsIgnoreCase("cmpassword")) {
- password = env.get(envName);
- if (cmdebug.equalsIgnoreCase("1"))
- System.out.println("cmpassword=" + password);
- } else if (envName.equalsIgnoreCase("cmserver")) {
- cmipaddress = env.get(envName);
- if (cmdebug.equalsIgnoreCase("1"))
- System.out.println("cmserver=" + cmipaddress);
- } else if (envName.equalsIgnoreCase("cmkey")) {
- key = env.get(envName);
- if (cmdebug.equalsIgnoreCase("1"))
- System.out.println("cmkey=" + key);
- } else if (envName.equalsIgnoreCase("cmdataformat")) {
- dataformat = env.get(envName);
- if (cmdebug.equalsIgnoreCase("1"))
- System.out.println("cmdataformat=" + dataformat);
- } else if (envName.equalsIgnoreCase("cmdebug")) {
- cmdebug = env.get(envName);
- if (cmdebug.equalsIgnoreCase("1"))
- System.out.println("cmdebug=" + cmdebug);
- }
- if (cmdebug.equalsIgnoreCase("1"))
- System.out.format("%s=%s%n", envName, env.get(envName));
- }
-
-
- }
-
- private String posttext(String url, String text) throws IOException {
- RequestBody body = RequestBody.create(text, JSONTXT);
- Request request = new Request.Builder().url(url).post(body).addHeader("Authorization", "Bearer " + this.token)
- .addHeader("Accept", "text/plain").addHeader("Content-Type", "text/plain").build();
- try (Response response = client.newCall(request).execute()) {
- return response.body().string();
- }
- }
-
- private String poststream(String url, String json) throws IOException {
- RequestBody body = RequestBody.create(json, JSONOCTET);
- Request request = new Request.Builder().url(url).post(body).addHeader("Authorization", "Bearer " + this.token)
- .addHeader("Accept", "application/json").addHeader("Content-Type", "application/octet-stream").build();
- try (Response response = client.newCall(request).execute()) {
- return response.body().string();
- }
- }
-
- private String getjson(String url) throws IOException {
- Request request = new Request.Builder().url(url).method("GET", null)
- .addHeader("Authorization", "Bearer " + this.token).addHeader("Accept", "application/json")
- .addHeader("Content-Type", "application/json").build();
- try (Response response = client.newCall(request).execute()) {
- return response.body().string();
- }
- }
-
- private String postjson(String url, String json) throws IOException {
- RequestBody body = RequestBody.create(json, JSON);
- Request request = new Request.Builder().url(url).post(body).addHeader("Authorization", "Bearer " + this.token)
- .addHeader("Accept", "application/json").addHeader("Content-Type", "application/json").build();
- try (Response response = client.newCall(request).execute()) {
- return response.body().string();
- }
- }
-
- /**
- * Returns an String that will be a JWT token to be used for REST calls
- * based on the refresh token.
- *
- * Note: This is using a Java KeyStore for authentication.
- *
- * @param keystorepwd
- * password to the java keystore
- * @param keystorelocation
- * location of javakeystore that contains certificates
- * @return string JWT token
- */
-
- public String getTokenFromRefresh(String keystorepwd, String keystorelocation) throws IOException {
-
- OkHttpClient client = getOkHttpClient(keystorepwd, this.cmipaddress, keystorelocation);
- MediaType mediaType = MediaType.parse("application/json");
- String grant_typetag = "{\"grant_type\":";
- String grant_type = "refresh_token";
- String refreshtokentag = "\"refresh_token\":";
-
- String authcall = grant_typetag + quote + grant_type + quote + comma + refreshtokentag + quote
- + this.refreshtoken + quote + " }";
-
- RequestBody body = RequestBody.create(authcall, mediaType);
- Request request = new Request.Builder().url("https://" + this.cmipaddress + "/api/v1/auth/tokens")
- .method("POST", body).addHeader("Content-Type", "application/json").build();
-
- Response response = client.newCall(request).execute();
- String returnvalue = response.body().string();
-
- String jwt = JsonPath.read(returnvalue.toString(), "$.jwt").toString();
-
- return jwt;
-
- }
-
- /**
- * Returns an String that will be a JWT token to be used for REST calls
- * based on the refresh token.
- *
- * Note: This is not using a Java KeyStore for authentication.
- *
- * @return string JWT token
- */
-
- public String getTokenFromRefresh() throws IOException {
-
- OkHttpClient client = getUnsafeOkHttpClient();
- MediaType mediaType = MediaType.parse("application/json");
-
- String grant_typetag = "{\"grant_type\":";
- String grant_type = "refresh_token";
- String refreshtokentag = "\"refresh_token\":";
-
- String authcall = grant_typetag + quote + grant_type + quote + comma + refreshtokentag + quote
- + this.refreshtoken + quote + " }";
-
- RequestBody body = RequestBody.create(authcall, mediaType);
- Request request = new Request.Builder().url("https://" + this.cmipaddress + "/api/v1/auth/tokens")
- .method("POST", body).addHeader("Content-Type", "application/json").build();
-
- Response response = client.newCall(request).execute();
- String returnvalue = response.body().string();
-
- String jwt = JsonPath.read(returnvalue.toString(), "$.jwt").toString();
-
- return jwt;
-
- }
-
- /**
- * Returns an String that will be a JWT token to be used for REST calls.
- *
- * Note: This is using a Java KeyStore for authentication.
- *
- * @param keystorepwd
- * password to the java keystore
- * @param keystorelocation
- * location of javakeystore that contains certificates
- * @return string JWT token
- */
- public String getToken(String keystorepwd, String keystorelocation) throws IOException {
-
- this.keystorepwd = keystorepwd;
- OkHttpClient client = getOkHttpClient(keystorepwd, this.cmipaddress, keystorelocation);
- MediaType mediaType = MediaType.parse("application/json");
-
- String grant_typetag = "{\"grant_type\":";
- String grant_type = "password";
- String passwordtag = "\"password\":";
- String usernametag = "\"username\":";
- String labels = "\"labels\": [\"myapp\",\"cli\"]}";
-
- String authcall = grant_typetag + quote + grant_type + quote + comma + usernametag + quote + this.username
- + quote + comma + passwordtag + quote + this.password + quote + comma + labels;
-
- RequestBody body = RequestBody.create(authcall, mediaType);
- Request request = new Request.Builder().url("https://" + this.cmipaddress + "/api/v1/auth/tokens")
- .method("POST", body).addHeader("Content-Type", "application/json").build();
-
- Response response = client.newCall(request).execute();
- String returnvalue = response.body().string();
-
- String jwt = JsonPath.read(returnvalue.toString(), "$.jwt").toString();
- this.refreshtoken = JsonPath.read(returnvalue.toString(), "$.refresh_token").toString();
-
- this.token = jwt;
- return jwt;
-
- }
-
- /**
- * Returns an String that will be a JWT token to be used for REST calls.
- *
- * Note: This is not using a Java KeyStore for authentication.
- *
- * @return string JWT token
- */
- public String getToken() throws IOException {
-
- OkHttpClient client = getUnsafeOkHttpClient();
-
- MediaType mediaType = MediaType.parse("application/json");
-
- String grant_typetag = "{\"grant_type\":";
- String grant_type = "password";
- String passwordtag = "\"password\":";
- String usernametag = "\"username\":";
- String labels = "\"labels\": [\"myapp\",\"cli\"]}";
-
- String authcall = grant_typetag + quote + grant_type + quote + comma + usernametag + quote + this.username
- + quote + comma + passwordtag + quote + this.password + quote + comma + labels;
- //System.out.println("auth call " + authcall);
- RequestBody body = RequestBody.create(authcall, mediaType);
-
- Request request = new Request.Builder().url("https://" + this.cmipaddress + "/api/v1/auth/tokens")
- .method("POST", body).addHeader("Content-Type", "application/json").build();
-
- Response response = client.newCall(request).execute();
- String returnvalue = response.body().string();
-
- String jwt = JsonPath.read(returnvalue.toString(), "$.jwt").toString();
- this.refreshtoken = JsonPath.read(returnvalue.toString(), "$.refresh_token").toString();
-
- this.token = jwt;
- return jwt;
-
- }
-
- private static KeyStore readKeyStore(String keystorepwd, String keystorelocation)
- throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException {
- KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
-
- char[] password = keystorepwd.toCharArray();
- java.io.FileInputStream fis = null;
- try {
-
- fis = new java.io.FileInputStream(keystorelocation);
- ks.load(fis, password);
- } finally {
- if (fis != null) {
- fis.close();
- }
- }
- return ks;
- }
-
- private static OkHttpClient getOkHttpClient(String pwd, String keymgrhostname, String keystorelocation) {
-
- try {
- TrustManagerFactory trustManagerFactory = TrustManagerFactory
- .getInstance(TrustManagerFactory.getDefaultAlgorithm());
-
- trustManagerFactory.init(readKeyStore(pwd, keystorelocation));
-
- X509TrustManager trustManager = (X509TrustManager) trustManagerFactory.getTrustManagers()[0];
- SSLContext sslContext = SSLContext.getInstance("TLS");
- sslContext.init(null, new TrustManager[] { trustManager }, null);
- java.security.cert.X509Certificate[] xcerts = trustManager.getAcceptedIssuers();
-
- for (int i = 0; i < xcerts.length; i++) {
- System.out.println(xcerts[i].getSigAlgName());
- System.out.println(xcerts[i].getType());
- // System.out.println(xcerts[i].getIssuerAlternativeNames().toString());
- System.out.println(xcerts[i].getIssuerDN().getName());
- System.out.println(xcerts[i].getSubjectDN().getName());
- System.out.println(xcerts[i].getPublicKey().getFormat().getBytes().toString());
- System.out.println(xcerts[i].getSerialNumber());
- }
- return new OkHttpClient.Builder().hostnameVerifier((hostname, session) -> {
- HostnameVerifier hv = HttpsURLConnection.getDefaultHostnameVerifier();
- /*
- * Never return true without verifying the hostname, otherwise
- * you will be vulnerable to man in the middle attacks.
- */
- boolean okhost = false;
-
- for (int i = 0; i < VALID_HOSTS.length; i++) {
- if (VALID_HOSTS[i].equalsIgnoreCase(keymgrhostname)) {
- okhost = true;
- break;
- }
- }
- return okhost;
- }).sslSocketFactory(sslContext.getSocketFactory(), trustManager).build();
-
- } catch (NoSuchAlgorithmException e) {
- e.printStackTrace();
- } catch (KeyStoreException e) {
- e.printStackTrace();
- } catch (KeyManagementException e) {
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- return null;
- }
-
- public static OkHttpClient.Builder enableTls12OVersion(OkHttpClient okHttpClient) {
- OkHttpClient.Builder client = okHttpClient.newBuilder();
- try {
- SSLContext sc = SSLContext.getInstance("TLSv1.2");
- sc.init(null, null, null);
- //sslSocketFactory = sc.getSocketFactory();
- // client.sslSocketFactory(new Tls12SocketFactory(sc.getSocketFactory()));
- client.sslSocketFactory(sc.getSocketFactory());
- ConnectionSpec connectionSpec = new ConnectionSpec.Builder(ConnectionSpec.MODERN_TLS)
- .tlsVersions(TlsVersion.TLS_1_2, TlsVersion.TLS_1_1, TlsVersion.TLS_1_0).cipherSuites(
- CipherSuite.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
- CipherSuite.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,
- CipherSuite.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
- CipherSuite.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA).build();
- List specs = new ArrayList<>();
- specs.add(connectionSpec);
- specs.add(ConnectionSpec.COMPATIBLE_TLS);
- specs.add(ConnectionSpec.CLEARTEXT);
- client.connectionSpecs(specs);
- } catch (Exception exc) {
- exc.printStackTrace();
- }
- return client;
- }
-
-
- private static OkHttpClient getUnsafeOkHttpClient() {
- try {
- // Create a trust manager that does not validate certificate chains
- final TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() {
- @Override
- public void checkClientTrusted(java.security.cert.X509Certificate[] chain, String authType)
- throws CertificateException {
- }
-
- @Override
- public void checkServerTrusted(java.security.cert.X509Certificate[] chain, String authType)
- throws CertificateException {
- }
-
- @Override
- public java.security.cert.X509Certificate[] getAcceptedIssuers() {
- return new java.security.cert.X509Certificate[] {};
- }
- } };
-
- // Install the all-trusting trust manager
- final SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
- //final SSLContext sslContext = SSLContext.getInstance("SSL");
- sslContext.init(null, trustAllCerts, new java.security.SecureRandom());
- // Create an ssl socket factory with our all-trusting manager
- final SSLSocketFactory sslSocketFactory = sslContext.getSocketFactory();
-//https://stackoverflow.com/questions/49980508/okhttp-sslhandshakeexception-ssl-handshake-aborted-failure-in-ssl-library-a-pro
-//https://square.github.io/okhttp/https/
- OkHttpClient.Builder builder = new OkHttpClient.Builder();
-
- builder.sslSocketFactory(sslSocketFactory, (X509TrustManager) trustAllCerts[0]);
- builder.hostnameVerifier(new HostnameVerifier() {
-
- @Override
- public boolean verify(String hostname, SSLSession session) {
- return true;
- }
- });
-
- OkHttpClient okHttpClient = builder.build();
- return okHttpClient;
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Will loop thru n number of times to test basic functionality for the
- * various methods implemented. See parameters below and examples for more
- * details.
- *
- * Examples: keyholder pwd MyAESEncryptionKey26 5 fpe 192.168.1.xxx
- * encrypt digit keyholder pwd myrsa-pub 5 rsa 192.168.1.xxx encrypt
- * alphabet keyholder pwd myrsa-pub 5 rsa 192.168.1.xxx encrypt
- * alphanumeric admin pwd! MyAESEncryptionKey26 10 fpe 192.168.1.xxx
- * encrypt digit admin pwd! hmacsha256-1 1 na 192.168.1.xxx mac alphabet
- * admin pwd! rsa-key5 1 na 192.168.1.xxx sign alphabet
- *
- * @param userid
- * the userid must be granted access to the key in CM.
- * @param password
- * the password of the user.
- * @param key
- * the key to be used. (Must be key name)
- * @param iterations
- * number of iterations to run. (Will use 25 bytes of random
- * data)
- * @param encmode
- * fpe,rsa,gcm,na
- * @param ciphertrustip
- * ciphertrust manger ip address
- * @param action
- * encrypt/decrypt/mac/macv/sign/signv
- * @param typeofdata
- * digit/alphabet/alphanumeric
- */
-
- public static void main(String[] args) throws Exception {
-
- if (args.length != 8) {
- System.err.println(
- "Usage: java CipherTrustManagerHelper2 userid pwd keyname iterations mode ciphertrustip [encrypt/decrypt/mac/macv/sign/signv] [digit/alphabet/alphanumeric] ");
- System.exit(-1);
- }
-
- CipherTrustManagerHelper awsresrest = new CipherTrustManagerHelper();
- awsresrest.username = args[0];
- awsresrest.password = args[1];
- awsresrest.key = args[2];
- int numberofrecords = Integer.parseInt(args[3]);
- awsresrest.cmipaddress = args[5];
- String action = args[6];
- String typeofdata = args[7];
-
- if (typeofdata.equalsIgnoreCase("digit"))
- awsresrest.dataformat = "digit";
- else if (typeofdata.equalsIgnoreCase("alphabet"))
- awsresrest.dataformat = "alphabet";
- else if (typeofdata.equalsIgnoreCase("alphanumeric"))
- awsresrest.dataformat = "alphanumeric";
- else
- throw new RuntimeException("valid values for data type are: digit,alphabet,alphanumeric");
-
- // String tkn = awsresrest.getToken("Vormetric123!",
- // "C:\\keystore\\cm_keystoreone");
- String tkn = awsresrest.getToken();
- Calendar calendar = Calendar.getInstance();
-
- // Get start time (this needs to be a global variable).
- Date startDate = calendar.getTime();
- System.out.println("key size is = " + awsresrest.getKeySize());
- // awsresrest.loop(args[4], numberofrecords, action);
-
- Calendar calendar2 = Calendar.getInstance();
-
- // Get start time (this needs to be a global variable).
- Date endDate = calendar2.getTime();
- long sumDate = endDate.getTime() - startDate.getTime();
- System.out.println("Total time " + sumDate);
- }
-
- /**
- * Returns an String that will either be a signed value or the string true
- * or false.
- *
- * Examples: awsresrest.cmRESTSign( "SHA1", "na",
- * "BGYUO07R7EBKYYMNGAIUAUPSJ","sign"); awsresrest.cmRESTSign( "SHA1",
- * "15e7eb8f1b0278583d71789a7aea05cbe1a041b2313f54c43f14a0c62b628175ece14bcfdecd47637049",
- * "BGYUO07R7EBKYYMNGAIUAUPSJ","signv");
- *
- * @param hashAlgo
- * the hash algorithum to be used. (SHA1, SHA-256, SHA-512,etc)
- * @param signature
- * the signature or na
- * @param data
- * the data to be signed
- * @param action
- * sign or signv.
- * @return either be a signed value or the string true or false
- */
-
- public String cmRESTSign(String hashAlgo, String signature, String data, String action) throws Exception {
-
- String value = null;
- int keysize = this.getKeySize() / 8;
-
- if (hashAlgo.equalsIgnoreCase("none") && data.length() > (keysize - encoding_parameters_length)) {
- throw new RuntimeException(
- "When using none for hashAlgo data size must be smaller than (keysize - encoding parm length)");
-
- }
-
- value = sign(hashAlgo, signature, data, action);
-
- return value;
-
- }
-
- /**
- * Returns an String that will either be a hash a value or the string true
- * or false.
- *
- * Examples: awsresrest.cmRESTMac( "na", "TIW58B91G25V3FN27491ACCTY","mac");
- * awsresrest.cmRESTMac("7c8842d207c1d73d21ae47b82cc18e6b03c48ef9ff1c3d27c5ccf2aa3d79f21e",
- * "TIW58B91G25V3FN27491ACCTY","macv");
- *
- * @param hash
- * the hash value must be either "na" or the actual hash value.
- * @param data
- * the data to be hashed
- * @param action
- * mac or macv.
- * @return either be a hash a value or the string true or false
- */
-
- public String cmRESTMac(String hash, String data, String action) throws Exception {
-
- String value = null;
-
- value = mac(hash, data, action);
-
- return value;
-
- }
-
- /**
- * Returns an String that will either be a be encrypted data or ciphertext.
- * Notes: 1.) when using rsa the key must be keyname of public key: example
- * rsakey-pub 2.) RSA Decrpyt will return values in base64 format. 3.) When
- * using gcm first part of encrypted data will include a tag and the tag
- * value
- *
- * Examples: awsresrest.cmRESTProtect( "gcm", text, "decrypt");
- * awsresrest.cmRESTProtect( "gcm", text, "encrypt");
- * awsresrest.cmRESTProtect( "rsa", text, "encrypt");
- * awsresrest.cmRESTProtect( "fpe", text, "encrypt");
- *
- * @param encmode
- * the encryption mode to be used for encryption or decrypt.
- * @param data
- * the data to be encrypted or the ciphertext in case of decrypt.
- * @param action
- * encrypt or decrypt.
- * @return either be encrypted data or ciphertext
- */
-
- public String cmRESTProtect(String encmode, String data, String action) throws Exception {
-
- String value = null;
-
- if (action.equalsIgnoreCase("encrypt")) {
- value = encrypt(encmode, data, action);
- } else if (action.equalsIgnoreCase("decrypt")) {
- value = decrypt(encmode, data, action);
- } else {
- System.out.println("Invalid action...... ");
-
- }
- return value;
-
- }
-
- private void loop(String encmode, int nbrofrecords, String action) throws Exception {
- String sensitive = null;
- String value;
-
- for (int i = 1; i <= nbrofrecords; i++) {
- if (this.dataformat.equalsIgnoreCase("digit"))
- sensitive = randomNumeric(25);
- else if (this.dataformat.equalsIgnoreCase("alphabet"))
- sensitive = randomAlpha(25);
- else
- sensitive = randomAlphaNumeric(25);
- System.out.println("original value = " + sensitive);
- if (action.equalsIgnoreCase("encrypt")) {
-
- value = this.cmRESTProtect(encmode, sensitive, action);
- System.out.println("return value from enc " + value);
-
- value = this.cmRESTProtect(encmode, value, "decrypt");
- System.out.println("return value from decrypt " + value);
- System.out.println("----------------------------------- ");
- // Following code is to test the refresh token logic
-
- /*
- * Thread.sleep(wait); System.out.println( "Thread '" +
- * Thread.currentThread().getName() +
- * "' is woken after sleeping for " + wait + " mseconds");
- */
- } else if (action.equalsIgnoreCase("mac")) {
- // admin Vormetric123! hmacsha256-1 1
- // 2c6e43fbf1bcf89ed75ee69280e5f53e78ffe8fb5591d1660d081a8613cf1f10
- // 192.168.1.xxx mac
- // admin Vormetric123! rsa-key5 1
- // 2c6e43fbf1bcf89ed75ee69280e5f53e78ffe8fb5591d1660d081a8613cf1f10
- // 192.168.1.xxx sign
- value = this.cmRESTMac("na", sensitive, action);
- System.out.println("mac " + value);
- value = this.cmRESTMac(value, sensitive, "macv");
- System.out.println("verify " + value);
-
- }
- // String hashAlgo , String signature ,String data, String action
- else if (action.equalsIgnoreCase("sign")) {
- value = this.cmRESTSign("SHA1", "na", sensitive, action);
- System.out.println("sign " + value);
- value = this.cmRESTSign("SHA1", value, sensitive, "signv");
- System.out.println("verify " + value);
-
- } else
- throw new RuntimeException("Invalid action code, valid values are: encrypt,sign,mac");
- }
-
- }
-
- private String buildSignVerifyURL(String hashAlgo, String signature, String action) {
- String url = null;
- if (action.equals("sign")) {
- url = "https://" + this.cmipaddress + "/api/v1/crypto/sign?keyName=" + this.key + "&hashAlgo=" + hashAlgo;
-
- } else if (action.equals("signv")) {
- url = "https://" + this.cmipaddress + "/api/v1/crypto/signv?keyName=" + this.key + "&hashAlgo=" + hashAlgo
- + "&signature=" + signature;
-
- } else {
- System.out.println("invalid action.... ");
- }
-
- return url;
- }
-
- private String buildURL() {
- String url = null;
- url = "https://" + this.cmipaddress + "/api/v1/vault/keys2/" + this.key;
-
- return url;
- }
-
- private String buildURL(String hash, String action) {
- String url = null;
- if (action.equals("mac")) {
- url = "https://" + this.cmipaddress + "/api/v1/crypto/mac?keyName=" + this.key;
-
- } else if (action.equals("macv")) {
- url = "https://" + this.cmipaddress + "/api/v1/crypto/macv?keyName=" + this.key + "&hash=" + hash;
-
- } else
- System.out.println("invalid action.... ");
- return url;
- }
-
- private String buildURL(String encmode, String sensitive, String action) {
-
- String hint = this.dataformat;
-
- String url = null;
- if (action.equalsIgnoreCase("encrypt")) {
- if (encmode.equals("fpe")) {
- if (hint.equalsIgnoreCase("digit") && sensitive.length() > digitblocklen)
- url = "https://" + this.cmipaddress + "/api/v1/crypto/hide2?keyName=" + this.key + "&version="
- + version + "&hint=" + hint + "&iv=" + numberPattern;
- else if (hint.equalsIgnoreCase("alphabet") && sensitive.length() > alphablocklen)
- url = "https://" + this.cmipaddress + "/api/v1/crypto/hide2?keyName=" + this.key + "&version="
- + version + "&hint=" + hint + "&iv=" + stringPattern;
- else if (hint.equalsIgnoreCase("alphanumeric") && sensitive.length() > alphablocklen)
- url = "https://" + this.cmipaddress + "/api/v1/crypto/hide2?keyName=" + this.key + "&version="
- + version + "&hint=" + hint + "&iv=" + combinedPattern;
- else
- url = "https://" + this.cmipaddress + "/api/v1/crypto/hide2?keyName=" + this.key + "&version="
- + version + "&hint=" + hint;
- } else {
- url = "https://" + this.cmipaddress + "/api/v1/crypto/encrypt";
-
- }
- } else if (action.equalsIgnoreCase("decrypt")) {
- if (encmode.equals("fpe")) {
- if (hint.equalsIgnoreCase("digit") && sensitive.length() > digitblocklen)
- url = "https://" + this.cmipaddress + "/api/v1/crypto/unhide2?keyName=" + this.key + "&version="
- + version + "&hint=" + hint + "&iv=" + numberPattern;
- else if (hint.equalsIgnoreCase("alphabet") && sensitive.length() > alphablocklen)
- url = "https://" + this.cmipaddress + "/api/v1/crypto/unhide2?keyName=" + this.key + "&version="
- + version + "&hint=" + hint + "&iv=" + stringPattern;
- else if (hint.equalsIgnoreCase("alphanumeric") && sensitive.length() > alphablocklen)
- url = "https://" + this.cmipaddress + "/api/v1/crypto/unhide2?keyName=" + this.key + "&version="
- + version + "&hint=" + hint + "&iv=" + combinedPattern;
- else
- url = "https://" + this.cmipaddress + "/api/v1/crypto/unhide2?keyName=" + this.key + "&version="
- + version + "&hint=" + hint;
- } else {
- url = "https://" + this.cmipaddress + "/api/v1/crypto/decrypt";
-
- }
- } else {
-
- System.out.println("invalid mode action provided ");
-
- }
- // System.out.println("url is " + url);
- return url;
- }
-
- private String getBody(String encmode, String sensitive, String action, String enctag) {
-
- String body = null;
-
- body = ciphertexttag + quote + sensitive + quote + comma + tagtag + quote + enctag + quote + comma + modetag
- + quote + encmode + quote + comma + idtag + quote + this.key + quote + comma + ivtag + quote + iv
- + quote + comma + aadtag + quote + aad + quote + endbracket;
-
- return body;
- }
-
- private String getBody(String encmode, String sensitive, String action) {
-
- String body = null;
- if (action.equalsIgnoreCase("encrypt")) {
- if (encmode.equals("fpe")) {
- body = sensitive;
- } else if (encmode.equals("rsa")) {
- byte[] dataBytes = sensitive.getBytes();
- String plaintextbase64 = Base64.getEncoder().encodeToString(dataBytes);
- body = plaintexttag + quote + plaintextbase64 + quote + comma + idtag + quote + this.key + quote
- + endbracket;
-
- } else {
- byte[] dataBytes = sensitive.getBytes();
- String plaintextbase64 = Base64.getEncoder().encodeToString(dataBytes);
- body = plaintexttag + quote + plaintextbase64 + quote + comma + modetag + quote + encmode + quote
- + comma + idtag + quote + this.key + quote + comma + ivtag + quote + iv + quote + comma + aadtag
- + quote + aad + quote + endbracket;
- }
-
- } else {
- if (encmode.equals("fpe")) {
- body = sensitive;
- } else if (encmode.equals("rsa")) {
- /*
- * byte[] dataBytes = sensitive.getBytes(); String
- * plaintextbase64 =
- * Base64.getEncoder().encodeToString(dataBytes);
- */
- // This example only works with passing in the public key name
- // for RSA keys.
- String keyprivate = this.key.substring(0, this.key.length() - 4);
- body = ciphertexttag + quote + sensitive + quote + comma + idtag + quote + keyprivate + quote + comma
- + typetag + name + comma + padtag + rsapad + endbracket;
-
- } else {
-
- body = ciphertexttag + quote + sensitive + quote + comma + modetag + quote + encmode + quote + comma
- + idtag + quote + this.key + quote + comma + ivtag + quote + iv + quote + comma + aadtag + quote
- + aad + quote + endbracket;
-
- }
- }
- // System.out.println(body);
- return body;
- }
-
- public int getKeySize() throws IOException {
- String results = null;
- int size = 0;
- results = this.getjson(buildURL());
-
- // System.out.println("value " + results);
- try {
- if (results.contains("Token is expired")) {
- System.out.println("get new token");
- this.token = getTokenFromRefresh();
- // Retry logic call post again.
-
- results = this.getjson(buildURL());
- }
- if (results.contains("Resource not found")) {
- System.out.println("Key not found");
-
- } else {
- results = JsonPath.read(results.toString(), "$.size").toString();
- Integer bigIntSize = new Integer(results);
- size = bigIntSize.intValue();
- }
- // System.out.println("key size = " + size);
- } catch (Exception e) {
-
- System.out.println(e.getMessage());
-
- String code = JsonPath.read(results.toString(), "$.code").toString();
- String msg = JsonPath.read(results.toString(), "$.codeDesc").toString();
- System.out.println("code " + code);
- System.out.println("code desc " + msg);
- StackTraceElement[] ste = e.getStackTrace();
- for (int j = 0; j < ste.length; j++) {
- System.out.println(ste[j]);
-
- }
-
- System.exit(-1);
- }
-
- return size;
-
- }
-
- private String sign(String hashAlgo, String signature, String sensitive, String action) throws Exception {
-
- String returnvalue = null;
- String ciphertext = null;
- String results = null;
- results = this.poststream(buildSignVerifyURL(hashAlgo, signature, action), sensitive);
-
- System.out.println("value " + results);
- try {
- if (results.contains("Token is expired")) {
- System.out.println("get new token");
- this.token = getTokenFromRefresh();
- // Retry logic call post again.
-
- results = this.poststream(buildSignVerifyURL(hashAlgo, signature, action), sensitive);
- }
- if (action.equals("sign")) {
- results = JsonPath.read(results.toString(), "$.data").toString();
- returnvalue = results;
- } else {
- ciphertext = JsonPath.read(results.toString(), "$.verified").toString();
- returnvalue = ciphertext;
- }
- // System.out.println("cipher text = " + ciphertext);
- } catch (Exception e) {
-
- System.out.println(e.getMessage());
-
- String code = JsonPath.read(results.toString(), "$.code").toString();
- String msg = JsonPath.read(results.toString(), "$.codeDesc").toString();
- System.out.println("code " + code);
- System.out.println("code desc " + msg);
- StackTraceElement[] ste = e.getStackTrace();
- for (int j = 0; j < ste.length; j++) {
- System.out.println(ste[j]);
-
- }
-
- System.exit(-1);
- }
-
- return returnvalue;
- }
-
- private String mac(String hash, String sensitive, String action) throws Exception {
-
- String returnvalue = null;
- String ciphertext = null;
- String results = null;
- results = this.poststream(buildURL(hash, action), sensitive);
-
- System.out.println("value " + results);
- try {
- if (results.contains("Token is expired")) {
- System.out.println("get new token");
- this.token = getTokenFromRefresh();
- // Retry logic call post again.
-
- results = this.postjson(buildURL(hash, action), sensitive);
- }
- if (action.equals("mac")) {
- results = JsonPath.read(results.toString(), "$.data").toString();
- returnvalue = results;
- } else {
- ciphertext = JsonPath.read(results.toString(), "$.verified").toString();
- returnvalue = ciphertext;
- }
- // System.out.println("cipher text = " + ciphertext);
- } catch (Exception e) {
-
- System.out.println(e.getMessage());
-
- String code = JsonPath.read(results.toString(), "$.code").toString();
- String msg = JsonPath.read(results.toString(), "$.codeDesc").toString();
- System.out.println("code " + code);
- System.out.println("code desc " + msg);
- StackTraceElement[] ste = e.getStackTrace();
- for (int j = 0; j < ste.length; j++) {
- System.out.println(ste[j]);
-
- }
-
- System.exit(-1);
- }
-
- return returnvalue;
- }
-
- private String decrypt(String encmode, String sensitive, String action) throws Exception {
-
- String returnvalue = null;
- String ciphertext = null;
- String enctag = null;
- String results = null;
- if (encmode.equals("fpe"))
- results = this.posttext(buildURL(encmode, sensitive, action), sensitive);
- else {
- if (sensitive.startsWith(filetagname)) {
- String str = sensitive.substring(filetagname.length() - 1);
- String parts[] = str.split(filetagsep);
- enctag = parts[0].replace(":", "");
- sensitive = parts[1];
- results = this.postjson(buildURL(encmode, sensitive, action),
- this.getBody(encmode, sensitive, action, enctag));
- } else {
- results = this.postjson(buildURL(encmode, sensitive, action), getBody(encmode, sensitive, action));
- }
- }
- // System.out.println("value " + results);
- try {
- if (results.contains("Token is expired")) {
- System.out.println("get new token");
- this.token = getTokenFromRefresh();
- // Retry logic call post again.
-
- if (encmode.equals("fpe"))
- results = this.posttext(buildURL(encmode, sensitive, action), sensitive);
- else {
- if (sensitive.startsWith(filetagname)) {
- String str = sensitive.substring(filetagname.length() - 1);
- String parts[] = str.split(filetagsep);
- enctag = parts[0];
- sensitive = parts[1];
-
- results = this.postjson(buildURL(encmode, sensitive, action),
- getBody(encmode, sensitive, action, enctag));
- } else {
- results = this.postjson(buildURL(encmode, sensitive, action),
- getBody(encmode, sensitive, action));
- }
- }
- }
-
- if (encmode.equals("fpe")) {
- results = JsonPath.read(results.toString(), "$.data").toString();
- returnvalue = results;
- } else if (encmode.equals("rsa")) {
- // byte[] bytes = results.getBytes("UTF-8");
- // byte[] decoded = Base64.getDecoder().decode(bytes);
- // System.out.println("orgi value new " + new String(decoded));
-
- // String plaintextbase64 = results.toString();
- // byte[] decryoriginaldata =
- // Base64.getDecoder().decode(plaintextbase64);
- // returnvalue = new String(new String(decoded));
- returnvalue = results;
-
- } else {
- String plaintextbase64 = JsonPath.read(results.toString(), "$.plaintext").toString();
- byte[] decryoriginaldata = Base64.getDecoder().decode(plaintextbase64);
- returnvalue = new String(decryoriginaldata);
-
- }
- // System.out.println("cipher text = " + ciphertext);
- } catch (Exception e) {
- // TODO: handle exception
- System.out.println(e.getMessage());
- if (!encmode.equals("rsa")) {
- String code = JsonPath.read(results.toString(), "$.code").toString();
- String msg = JsonPath.read(results.toString(), "$.codeDesc").toString();
- System.out.println("code " + code);
- System.out.println("code desc " + msg);
- StackTraceElement[] ste = e.getStackTrace();
- for (int j = 0; j < ste.length; j++) {
- System.out.println(ste[j]);
-
- }
- }
- System.exit(-1);
- }
-
- return returnvalue;
-
- }
-
- private String encrypt(String encmode, String sensitive, String action) throws Exception {
-
- String returnvalue = null;
- String ciphertext = null;
- String results = null;
- if (encmode.equals("fpe"))
- results = this.posttext(buildURL(encmode, sensitive, action), sensitive);
- else
- results = this.postjson(buildURL(encmode, sensitive, action), getBody(encmode, sensitive, action));
-
- //System.out.println("value " + results);
- try {
- if (results.contains("Token is expired")) {
- System.out.println("get new token");
- this.token = getTokenFromRefresh();
- // Retry logic call post again.
-
- if (encmode.equals("fpe"))
- results = this.posttext(buildURL(encmode, sensitive, action), sensitive);
- else
- results = this.postjson(buildURL(encmode, sensitive, action), getBody(encmode, sensitive, action));
-
- }
-
- if (encmode.equals("fpe")) {
- ciphertext = JsonPath.read(results.toString(), "$.data").toString();
- returnvalue = ciphertext;
- } else if (encmode.equals("rsa")) {
- ciphertext = JsonPath.read(results.toString(), "$.ciphertext").toString();
- returnvalue = ciphertext;
-
- } else {
- ciphertext = JsonPath.read(results.toString(), "$.ciphertext").toString();
- String tagtext = JsonPath.read(results.toString(), "$.tag").toString();
- returnvalue = filetagname + tagtext + filetagsep + ciphertext;
-
- }
- // System.out.println("cipher text = " + ciphertext);
- } catch (Exception e) {
- // TODO: handle exception
- System.out.println(e.getMessage());
-
- String code = JsonPath.read(results.toString(), "$.code").toString();
- String msg = JsonPath.read(results.toString(), "$.codeDesc").toString();
- System.out.println("code " + code);
- System.out.println("code desc " + msg);
- StackTraceElement[] ste = e.getStackTrace();
- for (int j = 0; j < ste.length; j++) {
- System.out.println(ste[j]);
-
- }
-
- System.exit(-1);
- }
-
- return returnvalue;
-
- }
-
- private static boolean isNumeric(String str) {
- for (char c : str.toCharArray()) {
-
- if (!Character.isDigit(c)) {
- return false;
- }
- }
- return true;
- }
-
- private static boolean isAlpha(String str) {
-
- for (char c : str.toCharArray()) {
- if (!Character.isAlphabetic(c)) {
- // if (!Character.isLetter(c)) {
- return false;
- }
- }
- return true;
-
- }
-
- private static String getSCUnique(String name) {
- StringBuffer returnvalue = new StringBuffer();
- HashMap hm = new HashMap();
- String specialCharacters = " !#$%&'()*+,-./:;<=>?@[]^_`{|}~";
- String str2[] = name.split("");
- int count = 0;
- for (int i = 0; i < str2.length; i++) {
- if (specialCharacters.contains(str2[i])) {
- count++;
- hm.put(str2[i], str2[i]);
- }
- }
-
- Set set = hm.entrySet();
- Iterator i = set.iterator();
-
- // Display elements
- while (i.hasNext()) {
- Map.Entry me = (Map.Entry) i.next();
- returnvalue.append(me.getKey());
- }
-
- return returnvalue.toString();
- }
-
- private static final String ALPHA_NUMERIC_STRING = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
- private static final String ALPHA = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
-
- private static String randomAlpha(int count) {
- StringBuilder builder = new StringBuilder();
- while (count-- != 0) {
- int character = (int) (Math.random() * ALPHA.length());
- builder.append(ALPHA.charAt(character));
- }
- return builder.toString();
- }
-
- private static String randomAlphaNumeric(int count) {
- StringBuilder builder = new StringBuilder();
- while (count-- != 0) {
- int character = (int) (Math.random() * ALPHA_NUMERIC_STRING.length());
- builder.append(ALPHA_NUMERIC_STRING.charAt(character));
- }
- return builder.toString();
- }
-
- private static final String NUMERIC_STRING = "0123456789";
-
- private static String randomNumeric(int count) {
- StringBuilder builder = new StringBuilder();
- while (count-- != 0) {
- int character = (int) (Math.random() * NUMERIC_STRING.length());
- builder.append(NUMERIC_STRING.charAt(character));
- }
- return builder.toString();
- }
-
-}
+package com.thales.cm.rest.cmhelper;
+
+import java.io.IOException;
+import java.security.KeyManagementException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.SSLSocketFactory;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509TrustManager;
+
+import com.jayway.jsonpath.JsonPath;
+
+import okhttp3.CipherSuite;
+import okhttp3.ConnectionSpec;
+import okhttp3.MediaType;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+import okhttp3.TlsVersion;
+
+/* This example helper class simplifies the usage of REST calls to the Thales CipherTrust Manager by
+ * exposing a few API's that shield some of the complexities of the json formating.
+ * This examples also shows how to work with refresh token and use retry logic when jwt expires.
+ * JWT duration = 300 seconds
+* Handles
+* 1.) encrypt/decrypt for gcm,rsa,fpe. (Note: RSA Decrypt returns values in base64 format)
+* 2.) mac/macv
+* 3.) sign/signv
+*
+* Note: This source code is only to be used for testing and proof of concepts. Not production ready code. Was not tested
+* for all possible data sizes and combinations of encryption algorithms and IV, etc.
+* Was tested with CM 2.1 & 2.2
+* Uses the okhttp3 4.7.2 library.
+*
+ */
+public class CipherTrustManagerHelper {
+ String cmdebug = "0";
+
+ public String token = null;
+ public String key = null;
+ public String cmipaddress;
+ public String refreshtoken;
+ public String keystorepwd;
+ public String username;
+ public String password;
+ public String dataformat;
+ public String iv ;
+
+ public static final String endbracket = "}";
+ public static final int digitblocklen = 56;
+ public static final int alphablocklen = 32;
+ public static final StringBuffer numberPattern = new StringBuffer(
+ "01234567890123456789012345678901024567896743678905435678");
+ public static final StringBuffer stringPattern = new StringBuffer("asdfghjklzxcvbnmqwertyuioplkjhgf");
+ public static final StringBuffer combinedPattern = new StringBuffer("abcdefghijklmnopqrstuvwxyz012345");
+ public static final String quote = "\"";
+ public static final String comma = ",";
+ public static final int wait = 60000;
+ public static final int encoding_parameters_length = 11;
+ public static final int version = 0;
+ public static final String versiontag = "\"version\":";
+ public static final String plaintexttag = "{\"plaintext\":";
+ public static final String tag = "kBr5A0fbPjPg7lS1bB6wfw==";
+ public static final String aadtag = "\"aad\":";
+ public static final String padtag = "\"pad\":";
+ public static final String rsapad = "\"oaep\"";
+ public static final String idtag = "\"id\":";
+ public static final String typetag = "\"type\":";
+ public static final String name = "\"name\"";
+ public static final String aad = "YXV0aGVudGljYXRl";
+ public static final String ciphertexttag = "{\"ciphertext\":";
+ public static final String tagtag = "\"tag\":";
+ public static final String ivtag = "\"iv\":";
+ public static final String modetag = "\"mode\":";
+ public static final String filetagname = "etag:";
+ public static final String filetagsep = "!";
+ private static final String[] VALID_HOSTS = { "ipaddress1", "ipaddress2", "fqdn1" };
+ public static final MediaType JSONOCTET = MediaType.get("application/octet-stream");
+ public static final MediaType JSON = MediaType.get("application/json; charset=utf-8");
+ public static final MediaType JSONTXT = MediaType.get("text/plain");
+
+ OkHttpClient client = getUnsafeOkHttpClient();
+
+ public CipherTrustManagerHelper() {
+ super();
+ Map env = System.getenv();
+ // System.out.println("name of logger" + log.getName());
+ // log.debug("this is a test");
+ // log.info("this is a test");
+ for (String envName : env.keySet()) {
+ if (envName.equalsIgnoreCase("cmuserid")) {
+ username = env.get(envName);
+ if (cmdebug.equalsIgnoreCase("1"))
+ System.out.println("cmuserid=" + username);
+ } else if (envName.equalsIgnoreCase("cmpassword")) {
+ password = env.get(envName);
+ if (cmdebug.equalsIgnoreCase("1"))
+ System.out.println("cmpassword=" + password);
+ } else if (envName.equalsIgnoreCase("cmserver")) {
+ cmipaddress = env.get(envName);
+ if (cmdebug.equalsIgnoreCase("1"))
+ System.out.println("cmserver=" + cmipaddress);
+ } else if (envName.equalsIgnoreCase("cmkey")) {
+ key = env.get(envName);
+ if (cmdebug.equalsIgnoreCase("1"))
+ System.out.println("cmkey=" + key);
+ } else if (envName.equalsIgnoreCase("cmdataformat")) {
+ dataformat = env.get(envName);
+ if (cmdebug.equalsIgnoreCase("1"))
+ System.out.println("cmdataformat=" + dataformat);
+ } else if (envName.equalsIgnoreCase("cmdebug")) {
+ cmdebug = env.get(envName);
+ if (cmdebug.equalsIgnoreCase("1"))
+ System.out.println("cmdebug=" + cmdebug);
+ } else if (envName.equalsIgnoreCase("iv")) {
+ iv = env.get(envName);
+ if (cmdebug.equalsIgnoreCase("1"))
+ System.out.println("iv=" + iv);
+ }
+ if (cmdebug.equalsIgnoreCase("1"))
+ System.out.format("%s=%s%n", envName, env.get(envName));
+ }
+
+
+ }
+
+ private String posttext(String url, String text) throws IOException {
+ RequestBody body = RequestBody.create(text, JSONTXT);
+ Request request = new Request.Builder().url(url).post(body).addHeader("Authorization", "Bearer " + this.token)
+ .addHeader("Accept", "text/plain").addHeader("Content-Type", "text/plain").build();
+ try (Response response = client.newCall(request).execute()) {
+ return response.body().string();
+ }
+ }
+
+ private String poststream(String url, String json) throws IOException {
+ RequestBody body = RequestBody.create(json, JSONOCTET);
+ Request request = new Request.Builder().url(url).post(body).addHeader("Authorization", "Bearer " + this.token)
+ .addHeader("Accept", "application/json").addHeader("Content-Type", "application/octet-stream").build();
+ try (Response response = client.newCall(request).execute()) {
+ return response.body().string();
+ }
+ }
+
+ private String getjson(String url) throws IOException {
+ Request request = new Request.Builder().url(url).method("GET", null)
+ .addHeader("Authorization", "Bearer " + this.token).addHeader("Accept", "application/json")
+ .addHeader("Content-Type", "application/json").build();
+ try (Response response = client.newCall(request).execute()) {
+ return response.body().string();
+ }
+ }
+
+ private String postjson(String url, String json) throws IOException {
+ RequestBody body = RequestBody.create(json, JSON);
+ Request request = new Request.Builder().url(url).post(body).addHeader("Authorization", "Bearer " + this.token)
+ .addHeader("Accept", "application/json").addHeader("Content-Type", "application/json").build();
+ try (Response response = client.newCall(request).execute()) {
+ return response.body().string();
+ }
+ }
+
+ /**
+ * Returns an String that will be a JWT token to be used for REST calls
+ * based on the refresh token.
+ *
+ * Note: This is using a Java KeyStore for authentication.
+ *
+ * @param keystorepwd
+ * password to the java keystore
+ * @param keystorelocation
+ * location of javakeystore that contains certificates
+ * @return string JWT token
+ */
+
+ public String getTokenFromRefresh(String keystorepwd, String keystorelocation) throws IOException {
+
+ OkHttpClient client = getOkHttpClient(keystorepwd, this.cmipaddress, keystorelocation);
+ MediaType mediaType = MediaType.parse("application/json");
+ String grant_typetag = "{\"grant_type\":";
+ String grant_type = "refresh_token";
+ String refreshtokentag = "\"refresh_token\":";
+
+ String authcall = grant_typetag + quote + grant_type + quote + comma + refreshtokentag + quote
+ + this.refreshtoken + quote + " }";
+
+ RequestBody body = RequestBody.create(authcall, mediaType);
+ Request request = new Request.Builder().url("https://" + this.cmipaddress + "/api/v1/auth/tokens")
+ .method("POST", body).addHeader("Content-Type", "application/json").build();
+
+ Response response = client.newCall(request).execute();
+ String returnvalue = response.body().string();
+
+ String jwt = JsonPath.read(returnvalue.toString(), "$.jwt").toString();
+
+ return jwt;
+
+ }
+
+ /**
+ * Returns an String that will be a JWT token to be used for REST calls
+ * based on the refresh token.
+ *
+ * Note: This is not using a Java KeyStore for authentication.
+ *
+ * @return string JWT token
+ */
+
+ public String getTokenFromRefresh() throws IOException {
+
+ OkHttpClient client = getUnsafeOkHttpClient();
+ MediaType mediaType = MediaType.parse("application/json");
+
+ String grant_typetag = "{\"grant_type\":";
+ String grant_type = "refresh_token";
+ String refreshtokentag = "\"refresh_token\":";
+
+ String authcall = grant_typetag + quote + grant_type + quote + comma + refreshtokentag + quote
+ + this.refreshtoken + quote + " }";
+
+ RequestBody body = RequestBody.create(authcall, mediaType);
+ Request request = new Request.Builder().url("https://" + this.cmipaddress + "/api/v1/auth/tokens")
+ .method("POST", body).addHeader("Content-Type", "application/json").build();
+
+ Response response = client.newCall(request).execute();
+ String returnvalue = response.body().string();
+
+ String jwt = JsonPath.read(returnvalue.toString(), "$.jwt").toString();
+
+ return jwt;
+
+ }
+
+ /**
+ * Returns an String that will be a JWT token to be used for REST calls.
+ *
+ * Note: This is using a Java KeyStore for authentication.
+ *
+ * @param keystorepwd
+ * password to the java keystore
+ * @param keystorelocation
+ * location of javakeystore that contains certificates
+ * @return string JWT token
+ */
+ public String getToken(String keystorepwd, String keystorelocation) throws IOException {
+
+ this.keystorepwd = keystorepwd;
+ OkHttpClient client = getOkHttpClient(keystorepwd, this.cmipaddress, keystorelocation);
+ MediaType mediaType = MediaType.parse("application/json");
+
+ String grant_typetag = "{\"grant_type\":";
+ String grant_type = "password";
+ String passwordtag = "\"password\":";
+ String usernametag = "\"username\":";
+ String labels = "\"labels\": [\"myapp\",\"cli\"]}";
+
+ String authcall = grant_typetag + quote + grant_type + quote + comma + usernametag + quote + this.username
+ + quote + comma + passwordtag + quote + this.password + quote + comma + labels;
+
+ RequestBody body = RequestBody.create(authcall, mediaType);
+ Request request = new Request.Builder().url("https://" + this.cmipaddress + "/api/v1/auth/tokens")
+ .method("POST", body).addHeader("Content-Type", "application/json").build();
+
+ Response response = client.newCall(request).execute();
+ String returnvalue = response.body().string();
+
+ String jwt = JsonPath.read(returnvalue.toString(), "$.jwt").toString();
+ this.refreshtoken = JsonPath.read(returnvalue.toString(), "$.refresh_token").toString();
+
+ this.token = jwt;
+ return jwt;
+
+ }
+
+ /**
+ * Returns an String that will be a JWT token to be used for REST calls.
+ *
+ * Note: This is not using a Java KeyStore for authentication.
+ *
+ * @return string JWT token
+ */
+ public String getToken() throws IOException {
+
+ OkHttpClient client = getUnsafeOkHttpClient();
+
+ MediaType mediaType = MediaType.parse("application/json");
+
+ String grant_typetag = "{\"grant_type\":";
+ String grant_type = "password";
+ String passwordtag = "\"password\":";
+ String usernametag = "\"username\":";
+ String labels = "\"labels\": [\"myapp\",\"cli\"]}";
+
+ String authcall = grant_typetag + quote + grant_type + quote + comma + usernametag + quote + this.username
+ + quote + comma + passwordtag + quote + this.password + quote + comma + labels;
+ //System.out.println("auth call " + authcall);
+ RequestBody body = RequestBody.create(authcall, mediaType);
+
+ Request request = new Request.Builder().url("https://" + this.cmipaddress + "/api/v1/auth/tokens")
+ .method("POST", body).addHeader("Content-Type", "application/json").build();
+
+ Response response = client.newCall(request).execute();
+ String returnvalue = response.body().string();
+
+ String jwt = JsonPath.read(returnvalue.toString(), "$.jwt").toString();
+ this.refreshtoken = JsonPath.read(returnvalue.toString(), "$.refresh_token").toString();
+
+ this.token = jwt;
+ return jwt;
+
+ }
+
+ private static KeyStore readKeyStore(String keystorepwd, String keystorelocation)
+ throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException {
+ KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
+
+ char[] password = keystorepwd.toCharArray();
+ java.io.FileInputStream fis = null;
+ try {
+
+ fis = new java.io.FileInputStream(keystorelocation);
+ ks.load(fis, password);
+ } finally {
+ if (fis != null) {
+ fis.close();
+ }
+ }
+ return ks;
+ }
+
+ private static OkHttpClient getOkHttpClient(String pwd, String keymgrhostname, String keystorelocation) {
+
+ try {
+ TrustManagerFactory trustManagerFactory = TrustManagerFactory
+ .getInstance(TrustManagerFactory.getDefaultAlgorithm());
+
+ trustManagerFactory.init(readKeyStore(pwd, keystorelocation));
+
+ X509TrustManager trustManager = (X509TrustManager) trustManagerFactory.getTrustManagers()[0];
+ SSLContext sslContext = SSLContext.getInstance("TLS");
+ sslContext.init(null, new TrustManager[] { trustManager }, null);
+ java.security.cert.X509Certificate[] xcerts = trustManager.getAcceptedIssuers();
+
+ for (int i = 0; i < xcerts.length; i++) {
+ System.out.println(xcerts[i].getSigAlgName());
+ System.out.println(xcerts[i].getType());
+ // System.out.println(xcerts[i].getIssuerAlternativeNames().toString());
+ System.out.println(xcerts[i].getIssuerDN().getName());
+ System.out.println(xcerts[i].getSubjectDN().getName());
+ System.out.println(xcerts[i].getPublicKey().getFormat().getBytes().toString());
+ System.out.println(xcerts[i].getSerialNumber());
+ }
+ return new OkHttpClient.Builder().hostnameVerifier((hostname, session) -> {
+ HostnameVerifier hv = HttpsURLConnection.getDefaultHostnameVerifier();
+ /*
+ * Never return true without verifying the hostname, otherwise
+ * you will be vulnerable to man in the middle attacks.
+ */
+ boolean okhost = false;
+
+ for (int i = 0; i < VALID_HOSTS.length; i++) {
+ if (VALID_HOSTS[i].equalsIgnoreCase(keymgrhostname)) {
+ okhost = true;
+ break;
+ }
+ }
+ return okhost;
+ }).sslSocketFactory(sslContext.getSocketFactory(), trustManager).build();
+
+ } catch (NoSuchAlgorithmException e) {
+ e.printStackTrace();
+ } catch (KeyStoreException e) {
+ e.printStackTrace();
+ } catch (KeyManagementException e) {
+ e.printStackTrace();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ return null;
+ }
+
+ public static OkHttpClient.Builder enableTls12OVersion(OkHttpClient okHttpClient) {
+ OkHttpClient.Builder client = okHttpClient.newBuilder();
+ try {
+ SSLContext sc = SSLContext.getInstance("TLSv1.2");
+ sc.init(null, null, null);
+ //sslSocketFactory = sc.getSocketFactory();
+ // client.sslSocketFactory(new Tls12SocketFactory(sc.getSocketFactory()));
+ client.sslSocketFactory(sc.getSocketFactory());
+ ConnectionSpec connectionSpec = new ConnectionSpec.Builder(ConnectionSpec.MODERN_TLS)
+ .tlsVersions(TlsVersion.TLS_1_2, TlsVersion.TLS_1_1, TlsVersion.TLS_1_0).cipherSuites(
+ CipherSuite.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+ CipherSuite.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,
+ CipherSuite.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+ CipherSuite.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA).build();
+ List specs = new ArrayList<>();
+ specs.add(connectionSpec);
+ specs.add(ConnectionSpec.COMPATIBLE_TLS);
+ specs.add(ConnectionSpec.CLEARTEXT);
+ client.connectionSpecs(specs);
+ } catch (Exception exc) {
+ exc.printStackTrace();
+ }
+ return client;
+ }
+
+
+ private static OkHttpClient getUnsafeOkHttpClient() {
+ try {
+ // Create a trust manager that does not validate certificate chains
+ final TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() {
+ @Override
+ public void checkClientTrusted(java.security.cert.X509Certificate[] chain, String authType)
+ throws CertificateException {
+ }
+
+ @Override
+ public void checkServerTrusted(java.security.cert.X509Certificate[] chain, String authType)
+ throws CertificateException {
+ }
+
+ @Override
+ public java.security.cert.X509Certificate[] getAcceptedIssuers() {
+ return new java.security.cert.X509Certificate[] {};
+ }
+ } };
+
+ // Install the all-trusting trust manager
+ final SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
+ //final SSLContext sslContext = SSLContext.getInstance("SSL");
+ sslContext.init(null, trustAllCerts, new java.security.SecureRandom());
+ // Create an ssl socket factory with our all-trusting manager
+ final SSLSocketFactory sslSocketFactory = sslContext.getSocketFactory();
+//https://stackoverflow.com/questions/49980508/okhttp-sslhandshakeexception-ssl-handshake-aborted-failure-in-ssl-library-a-pro
+//https://square.github.io/okhttp/https/
+ OkHttpClient.Builder builder = new OkHttpClient.Builder();
+
+ builder.sslSocketFactory(sslSocketFactory, (X509TrustManager) trustAllCerts[0]);
+ builder.hostnameVerifier(new HostnameVerifier() {
+
+ @Override
+ public boolean verify(String hostname, SSLSession session) {
+ return true;
+ }
+ });
+
+ OkHttpClient okHttpClient = builder.build();
+ return okHttpClient;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Will loop thru n number of times to test basic functionality for the
+ * various methods implemented. See parameters below and examples for more
+ * details.
+ *
+ * Examples: keyholder pwd MyAESEncryptionKey26 5 fpe 192.168.1.xxx
+ * encrypt digit keyholder pwd myrsa-pub 5 rsa 192.168.1.xxx encrypt
+ * alphabet keyholder pwd myrsa-pub 5 rsa 192.168.1.xxx encrypt
+ * alphanumeric admin pwd! MyAESEncryptionKey26 10 fpe 192.168.1.xxx
+ * encrypt digit admin pwd! hmacsha256-1 1 na 192.168.1.xxx mac alphabet
+ * admin pwd! rsa-key5 1 na 192.168.1.xxx sign alphabet
+ *
+ * @param userid
+ * the userid must be granted access to the key in CM.
+ * @param password
+ * the password of the user.
+ * @param key
+ * the key to be used. (Must be key name)
+ * @param iterations
+ * number of iterations to run. (Will use 25 bytes of random
+ * data)
+ * @param encmode
+ * fpe,rsa,gcm,na
+ * @param ciphertrustip
+ * ciphertrust manger ip address
+ * @param action
+ * encrypt/decrypt/mac/macv/sign/signv
+ * @param typeofdata
+ * digit/alphabet/alphanumeric
+ */
+
+ public static void main(String[] args) throws Exception {
+
+ if (args.length != 8) {
+ System.err.println(
+ "Usage: java CipherTrustManagerHelper2 userid pwd keyname iterations mode ciphertrustip [encrypt/decrypt/mac/macv/sign/signv] [digit/alphabet/alphanumeric] ");
+ System.exit(-1);
+ }
+
+ CipherTrustManagerHelper awsresrest = new CipherTrustManagerHelper();
+ awsresrest.username = args[0];
+ awsresrest.password = args[1];
+ awsresrest.key = args[2];
+ int numberofrecords = Integer.parseInt(args[3]);
+ awsresrest.cmipaddress = args[5];
+ String action = args[6];
+ String typeofdata = args[7];
+
+ if (typeofdata.equalsIgnoreCase("digit"))
+ awsresrest.dataformat = "digit";
+ else if (typeofdata.equalsIgnoreCase("alphabet"))
+ awsresrest.dataformat = "alphabet";
+ else if (typeofdata.equalsIgnoreCase("alphanumeric"))
+ awsresrest.dataformat = "alphanumeric";
+ else
+ throw new RuntimeException("valid values for data type are: digit,alphabet,alphanumeric");
+
+ // String tkn = awsresrest.getToken("Vormetric123!",
+ // "C:\\keystore\\cm_keystoreone");
+ String tkn = awsresrest.getToken();
+ Calendar calendar = Calendar.getInstance();
+
+ // Get start time (this needs to be a global variable).
+ Date startDate = calendar.getTime();
+ System.out.println("key size is = " + awsresrest.getKeySize());
+ // awsresrest.loop(args[4], numberofrecords, action);
+
+ Calendar calendar2 = Calendar.getInstance();
+
+ // Get start time (this needs to be a global variable).
+ Date endDate = calendar2.getTime();
+ long sumDate = endDate.getTime() - startDate.getTime();
+ System.out.println("Total time " + sumDate);
+ }
+
+ /**
+ * Returns an String that will either be a signed value or the string true
+ * or false.
+ *
+ * Examples: awsresrest.cmRESTSign( "SHA1", "na",
+ * "BGYUO07R7EBKYYMNGAIUAUPSJ","sign"); awsresrest.cmRESTSign( "SHA1",
+ * "15e7eb8f1b0278583d71789a7aea05cbe1a041b2313f54c43f14a0c62b628175ece14bcfdecd47637049",
+ * "BGYUO07R7EBKYYMNGAIUAUPSJ","signv");
+ *
+ * @param hashAlgo
+ * the hash algorithum to be used. (SHA1, SHA-256, SHA-512,etc)
+ * @param signature
+ * the signature or na
+ * @param data
+ * the data to be signed
+ * @param action
+ * sign or signv.
+ * @return either be a signed value or the string true or false
+ */
+
+ public String cmRESTSign(String hashAlgo, String signature, String data, String action) throws Exception {
+
+ String value = null;
+ int keysize = this.getKeySize() / 8;
+
+ if (hashAlgo.equalsIgnoreCase("none") && data.length() > (keysize - encoding_parameters_length)) {
+ throw new RuntimeException(
+ "When using none for hashAlgo data size must be smaller than (keysize - encoding parm length)");
+
+ }
+
+ value = sign(hashAlgo, signature, data, action);
+
+ return value;
+
+ }
+
+ /**
+ * Returns an String that will either be a hash a value or the string true
+ * or false.
+ *
+ * Examples: awsresrest.cmRESTMac( "na", "TIW58B91G25V3FN27491ACCTY","mac");
+ * awsresrest.cmRESTMac("7c8842d207c1d73d21ae47b82cc18e6b03c48ef9ff1c3d27c5ccf2aa3d79f21e",
+ * "TIW58B91G25V3FN27491ACCTY","macv");
+ *
+ * @param hash
+ * the hash value must be either "na" or the actual hash value.
+ * @param data
+ * the data to be hashed
+ * @param action
+ * mac or macv.
+ * @return either be a hash a value or the string true or false
+ */
+
+ public String cmRESTMac(String hash, String data, String action) throws Exception {
+
+ String value = null;
+
+ value = mac(hash, data, action);
+
+ return value;
+
+ }
+
+ /**
+ * Returns an String that will either be a be encrypted data or ciphertext.
+ * Notes: 1.) when using rsa the key must be keyname of public key: example
+ * rsakey-pub 2.) RSA Decrpyt will return values in base64 format. 3.) When
+ * using gcm first part of encrypted data will include a tag and the tag
+ * value
+ *
+ * Examples: awsresrest.cmRESTProtect( "gcm", text, "decrypt");
+ * awsresrest.cmRESTProtect( "gcm", text, "encrypt");
+ * awsresrest.cmRESTProtect( "rsa", text, "encrypt");
+ * awsresrest.cmRESTProtect( "fpe", text, "encrypt");
+ *
+ * @param encmode
+ * the encryption mode to be used for encryption or decrypt.
+ * @param data
+ * the data to be encrypted or the ciphertext in case of decrypt.
+ * @param action
+ * encrypt or decrypt.
+ * @return either be encrypted data or ciphertext
+ */
+
+ public String cmRESTProtect(String encmode, String data, String action) throws Exception {
+
+ String value = null;
+
+ if (action.equalsIgnoreCase("encrypt")) {
+ value = encrypt(encmode, data, action);
+ } else if (action.equalsIgnoreCase("decrypt")) {
+ value = decrypt(encmode, data, action);
+ } else {
+ System.out.println("Invalid action...... ");
+
+ }
+ return value;
+
+ }
+
+ private void loop(String encmode, int nbrofrecords, String action) throws Exception {
+ String sensitive = null;
+ String value;
+
+ for (int i = 1; i <= nbrofrecords; i++) {
+ if (this.dataformat.equalsIgnoreCase("digit"))
+ sensitive = randomNumeric(25);
+ else if (this.dataformat.equalsIgnoreCase("alphabet"))
+ sensitive = randomAlpha(25);
+ else
+ sensitive = randomAlphaNumeric(25);
+ System.out.println("original value = " + sensitive);
+ if (action.equalsIgnoreCase("encrypt")) {
+
+ value = this.cmRESTProtect(encmode, sensitive, action);
+ System.out.println("return value from enc " + value);
+
+ value = this.cmRESTProtect(encmode, value, "decrypt");
+ System.out.println("return value from decrypt " + value);
+ System.out.println("----------------------------------- ");
+ // Following code is to test the refresh token logic
+
+ /*
+ * Thread.sleep(wait); System.out.println( "Thread '" +
+ * Thread.currentThread().getName() +
+ * "' is woken after sleeping for " + wait + " mseconds");
+ */
+ } else if (action.equalsIgnoreCase("mac")) {
+ // admin Vormetric123! hmacsha256-1 1
+ // 2c6e43fbf1bcf89ed75ee69280e5f53e78ffe8fb5591d1660d081a8613cf1f10
+ // 192.168.1.xxx mac
+ // admin Vormetric123! rsa-key5 1
+ // 2c6e43fbf1bcf89ed75ee69280e5f53e78ffe8fb5591d1660d081a8613cf1f10
+ // 192.168.1.xxx sign
+ value = this.cmRESTMac("na", sensitive, action);
+ System.out.println("mac " + value);
+ value = this.cmRESTMac(value, sensitive, "macv");
+ System.out.println("verify " + value);
+
+ }
+ // String hashAlgo , String signature ,String data, String action
+ else if (action.equalsIgnoreCase("sign")) {
+ value = this.cmRESTSign("SHA1", "na", sensitive, action);
+ System.out.println("sign " + value);
+ value = this.cmRESTSign("SHA1", value, sensitive, "signv");
+ System.out.println("verify " + value);
+
+ } else
+ throw new RuntimeException("Invalid action code, valid values are: encrypt,sign,mac");
+ }
+
+ }
+
+ private String buildSignVerifyURL(String hashAlgo, String signature, String action) {
+ String url = null;
+ if (action.equals("sign")) {
+ url = "https://" + this.cmipaddress + "/api/v1/crypto/sign?keyName=" + this.key + "&hashAlgo=" + hashAlgo;
+
+ } else if (action.equals("signv")) {
+ url = "https://" + this.cmipaddress + "/api/v1/crypto/signv?keyName=" + this.key + "&hashAlgo=" + hashAlgo
+ + "&signature=" + signature;
+
+ } else {
+ System.out.println("invalid action.... ");
+ }
+
+ return url;
+ }
+
+ private String buildURL() {
+ String url = null;
+ url = "https://" + this.cmipaddress + "/api/v1/vault/keys2/" + this.key;
+
+ return url;
+ }
+
+ private String buildURL(String hash, String action) {
+ String url = null;
+ if (action.equals("mac")) {
+ url = "https://" + this.cmipaddress + "/api/v1/crypto/mac?keyName=" + this.key;
+
+ } else if (action.equals("macv")) {
+ url = "https://" + this.cmipaddress + "/api/v1/crypto/macv?keyName=" + this.key + "&hash=" + hash;
+
+ } else
+ System.out.println("invalid action.... ");
+ return url;
+ }
+
+ private String buildURL(String encmode, String sensitive, String action) {
+
+ String hint = this.dataformat;
+
+ String url = null;
+ if (action.equalsIgnoreCase("encrypt")) {
+ if (encmode.equals("fpe")) {
+ if (hint.equalsIgnoreCase("digit") && sensitive.length() > digitblocklen)
+ url = "https://" + this.cmipaddress + "/api/v1/crypto/hide2?keyName=" + this.key + "&version="
+ + version + "&hint=" + hint + "&iv=" + numberPattern;
+ else if (hint.equalsIgnoreCase("alphabet") && sensitive.length() > alphablocklen)
+ url = "https://" + this.cmipaddress + "/api/v1/crypto/hide2?keyName=" + this.key + "&version="
+ + version + "&hint=" + hint + "&iv=" + stringPattern;
+ else if (hint.equalsIgnoreCase("alphanumeric") && sensitive.length() > alphablocklen)
+ url = "https://" + this.cmipaddress + "/api/v1/crypto/hide2?keyName=" + this.key + "&version="
+ + version + "&hint=" + hint + "&iv=" + combinedPattern;
+ else
+ url = "https://" + this.cmipaddress + "/api/v1/crypto/hide2?keyName=" + this.key + "&version="
+ + version + "&hint=" + hint;
+ } else {
+ url = "https://" + this.cmipaddress + "/api/v1/crypto/encrypt";
+
+ }
+ } else if (action.equalsIgnoreCase("decrypt")) {
+ if (encmode.equals("fpe")) {
+ if (hint.equalsIgnoreCase("digit") && sensitive.length() > digitblocklen)
+ url = "https://" + this.cmipaddress + "/api/v1/crypto/unhide2?keyName=" + this.key + "&version="
+ + version + "&hint=" + hint + "&iv=" + numberPattern;
+ else if (hint.equalsIgnoreCase("alphabet") && sensitive.length() > alphablocklen)
+ url = "https://" + this.cmipaddress + "/api/v1/crypto/unhide2?keyName=" + this.key + "&version="
+ + version + "&hint=" + hint + "&iv=" + stringPattern;
+ else if (hint.equalsIgnoreCase("alphanumeric") && sensitive.length() > alphablocklen)
+ url = "https://" + this.cmipaddress + "/api/v1/crypto/unhide2?keyName=" + this.key + "&version="
+ + version + "&hint=" + hint + "&iv=" + combinedPattern;
+ else
+ url = "https://" + this.cmipaddress + "/api/v1/crypto/unhide2?keyName=" + this.key + "&version="
+ + version + "&hint=" + hint;
+ } else {
+ url = "https://" + this.cmipaddress + "/api/v1/crypto/decrypt";
+
+ }
+ } else {
+
+ System.out.println("invalid mode action provided ");
+
+ }
+ // System.out.println("url is " + url);
+ return url;
+ }
+
+ private String getBody(String encmode, String sensitive, String action, String enctag) {
+
+ String body = null;
+
+ body = ciphertexttag + quote + sensitive + quote + comma + tagtag + quote + enctag + quote + comma + modetag
+ + quote + encmode + quote + comma + idtag + quote + this.key + quote + comma + ivtag + quote + this.iv
+ + quote + comma + aadtag + quote + aad + quote + endbracket;
+
+ return body;
+ }
+
+ private String getBody(String encmode, String sensitive, String action) {
+
+ String body = null;
+ if (action.equalsIgnoreCase("encrypt")) {
+ if (encmode.equals("fpe")) {
+ body = sensitive;
+ } else if (encmode.equals("rsa")) {
+ byte[] dataBytes = sensitive.getBytes();
+ String plaintextbase64 = Base64.getEncoder().encodeToString(dataBytes);
+ body = plaintexttag + quote + plaintextbase64 + quote + comma + idtag + quote + this.key + quote
+ + endbracket;
+
+ } else {
+ byte[] dataBytes = sensitive.getBytes();
+ String plaintextbase64 = Base64.getEncoder().encodeToString(dataBytes);
+ body = plaintexttag + quote + plaintextbase64 + quote + comma + modetag + quote + encmode + quote
+ + comma + idtag + quote + this.key + quote + comma + ivtag + quote + this.iv + quote + comma + aadtag
+ + quote + aad + quote + endbracket;
+ }
+
+ } else {
+ if (encmode.equals("fpe")) {
+ body = sensitive;
+ } else if (encmode.equals("rsa")) {
+ /*
+ * byte[] dataBytes = sensitive.getBytes(); String
+ * plaintextbase64 =
+ * Base64.getEncoder().encodeToString(dataBytes);
+ */
+ // This example only works with passing in the public key name
+ // for RSA keys.
+ String keyprivate = this.key.substring(0, this.key.length() - 4);
+ body = ciphertexttag + quote + sensitive + quote + comma + idtag + quote + keyprivate + quote + comma
+ + typetag + name + comma + padtag + rsapad + endbracket;
+
+ } else {
+
+ body = ciphertexttag + quote + sensitive + quote + comma + modetag + quote + encmode + quote + comma
+ + idtag + quote + this.key + quote + comma + ivtag + quote + this.iv + quote + comma + aadtag + quote
+ + aad + quote + endbracket;
+
+ }
+ }
+ // System.out.println(body);
+ return body;
+ }
+
+ public int getKeySize() throws IOException {
+ String results = null;
+ int size = 0;
+ results = this.getjson(buildURL());
+
+ // System.out.println("value " + results);
+ try {
+ if (results.contains("Token is expired")) {
+ System.out.println("get new token");
+ this.token = getTokenFromRefresh();
+ // Retry logic call post again.
+
+ results = this.getjson(buildURL());
+ }
+ if (results.contains("Resource not found")) {
+ System.out.println("Key not found");
+
+ } else {
+ results = JsonPath.read(results.toString(), "$.size").toString();
+ Integer bigIntSize = new Integer(results);
+ size = bigIntSize.intValue();
+ }
+ // System.out.println("key size = " + size);
+ } catch (Exception e) {
+
+ System.out.println(e.getMessage());
+
+ String code = JsonPath.read(results.toString(), "$.code").toString();
+ String msg = JsonPath.read(results.toString(), "$.codeDesc").toString();
+ System.out.println("code " + code);
+ System.out.println("code desc " + msg);
+ StackTraceElement[] ste = e.getStackTrace();
+ for (int j = 0; j < ste.length; j++) {
+ System.out.println(ste[j]);
+
+ }
+
+ System.exit(-1);
+ }
+
+ return size;
+
+ }
+
+ private String sign(String hashAlgo, String signature, String sensitive, String action) throws Exception {
+
+ String returnvalue = null;
+ String ciphertext = null;
+ String results = null;
+ results = this.poststream(buildSignVerifyURL(hashAlgo, signature, action), sensitive);
+
+ System.out.println("value " + results);
+ try {
+ if (results.contains("Token is expired")) {
+ System.out.println("get new token");
+ this.token = getTokenFromRefresh();
+ // Retry logic call post again.
+
+ results = this.poststream(buildSignVerifyURL(hashAlgo, signature, action), sensitive);
+ }
+ if (action.equals("sign")) {
+ results = JsonPath.read(results.toString(), "$.data").toString();
+ returnvalue = results;
+ } else {
+ ciphertext = JsonPath.read(results.toString(), "$.verified").toString();
+ returnvalue = ciphertext;
+ }
+ // System.out.println("cipher text = " + ciphertext);
+ } catch (Exception e) {
+
+ System.out.println(e.getMessage());
+
+ String code = JsonPath.read(results.toString(), "$.code").toString();
+ String msg = JsonPath.read(results.toString(), "$.codeDesc").toString();
+ System.out.println("code " + code);
+ System.out.println("code desc " + msg);
+ StackTraceElement[] ste = e.getStackTrace();
+ for (int j = 0; j < ste.length; j++) {
+ System.out.println(ste[j]);
+
+ }
+
+ System.exit(-1);
+ }
+
+ return returnvalue;
+ }
+
+ private String mac(String hash, String sensitive, String action) throws Exception {
+
+ String returnvalue = null;
+ String ciphertext = null;
+ String results = null;
+ results = this.poststream(buildURL(hash, action), sensitive);
+
+ System.out.println("value " + results);
+ try {
+ if (results.contains("Token is expired")) {
+ System.out.println("get new token");
+ this.token = getTokenFromRefresh();
+ // Retry logic call post again.
+
+ results = this.postjson(buildURL(hash, action), sensitive);
+ }
+ if (action.equals("mac")) {
+ results = JsonPath.read(results.toString(), "$.data").toString();
+ returnvalue = results;
+ } else {
+ ciphertext = JsonPath.read(results.toString(), "$.verified").toString();
+ returnvalue = ciphertext;
+ }
+ // System.out.println("cipher text = " + ciphertext);
+ } catch (Exception e) {
+
+ System.out.println(e.getMessage());
+
+ String code = JsonPath.read(results.toString(), "$.code").toString();
+ String msg = JsonPath.read(results.toString(), "$.codeDesc").toString();
+ System.out.println("code " + code);
+ System.out.println("code desc " + msg);
+ StackTraceElement[] ste = e.getStackTrace();
+ for (int j = 0; j < ste.length; j++) {
+ System.out.println(ste[j]);
+
+ }
+
+ System.exit(-1);
+ }
+
+ return returnvalue;
+ }
+
+ private String decrypt(String encmode, String sensitive, String action) throws Exception {
+
+ String returnvalue = null;
+ String ciphertext = null;
+ String enctag = null;
+ String results = null;
+ if (encmode.equals("fpe"))
+ results = this.posttext(buildURL(encmode, sensitive, action), sensitive);
+ else {
+ if (sensitive.startsWith(filetagname)) {
+ String str = sensitive.substring(filetagname.length() - 1);
+ String parts[] = str.split(filetagsep);
+ enctag = parts[0].replace(":", "");
+ sensitive = parts[1];
+ results = this.postjson(buildURL(encmode, sensitive, action),
+ this.getBody(encmode, sensitive, action, enctag));
+ } else {
+ results = this.postjson(buildURL(encmode, sensitive, action), getBody(encmode, sensitive, action));
+ }
+ }
+ // System.out.println("value " + results);
+ try {
+ if (results.contains("Token is expired")) {
+ System.out.println("get new token");
+ this.token = getTokenFromRefresh();
+ // Retry logic call post again.
+
+ if (encmode.equals("fpe"))
+ results = this.posttext(buildURL(encmode, sensitive, action), sensitive);
+ else {
+ if (sensitive.startsWith(filetagname)) {
+ String str = sensitive.substring(filetagname.length() - 1);
+ String parts[] = str.split(filetagsep);
+ enctag = parts[0];
+ sensitive = parts[1];
+
+ results = this.postjson(buildURL(encmode, sensitive, action),
+ getBody(encmode, sensitive, action, enctag));
+ } else {
+ results = this.postjson(buildURL(encmode, sensitive, action),
+ getBody(encmode, sensitive, action));
+ }
+ }
+ }
+
+ if (encmode.equals("fpe")) {
+ results = JsonPath.read(results.toString(), "$.data").toString();
+ returnvalue = results;
+ } else if (encmode.equals("rsa")) {
+ // byte[] bytes = results.getBytes("UTF-8");
+ // byte[] decoded = Base64.getDecoder().decode(bytes);
+ // System.out.println("orgi value new " + new String(decoded));
+
+ // String plaintextbase64 = results.toString();
+ // byte[] decryoriginaldata =
+ // Base64.getDecoder().decode(plaintextbase64);
+ // returnvalue = new String(new String(decoded));
+ returnvalue = results;
+
+ } else {
+ String plaintextbase64 = JsonPath.read(results.toString(), "$.plaintext").toString();
+ byte[] decryoriginaldata = Base64.getDecoder().decode(plaintextbase64);
+ returnvalue = new String(decryoriginaldata);
+
+ }
+ // System.out.println("cipher text = " + ciphertext);
+ } catch (Exception e) {
+ // TODO: handle exception
+ System.out.println(e.getMessage());
+ if (!encmode.equals("rsa")) {
+ String code = JsonPath.read(results.toString(), "$.code").toString();
+ String msg = JsonPath.read(results.toString(), "$.codeDesc").toString();
+ System.out.println("code " + code);
+ System.out.println("code desc " + msg);
+ StackTraceElement[] ste = e.getStackTrace();
+ for (int j = 0; j < ste.length; j++) {
+ System.out.println(ste[j]);
+
+ }
+ }
+ System.exit(-1);
+ }
+
+ return returnvalue;
+
+ }
+
+ private String encrypt(String encmode, String sensitive, String action) throws Exception {
+
+ String returnvalue = null;
+ String ciphertext = null;
+ String results = null;
+ if (encmode.equals("fpe"))
+ results = this.posttext(buildURL(encmode, sensitive, action), sensitive);
+ else
+ results = this.postjson(buildURL(encmode, sensitive, action), getBody(encmode, sensitive, action));
+
+ //System.out.println("value " + results);
+ try {
+ if (results.contains("Token is expired")) {
+ System.out.println("get new token");
+ this.token = getTokenFromRefresh();
+ // Retry logic call post again.
+
+ if (encmode.equals("fpe"))
+ results = this.posttext(buildURL(encmode, sensitive, action), sensitive);
+ else
+ results = this.postjson(buildURL(encmode, sensitive, action), getBody(encmode, sensitive, action));
+
+ }
+
+ if (encmode.equals("fpe")) {
+ ciphertext = JsonPath.read(results.toString(), "$.data").toString();
+ returnvalue = ciphertext;
+ } else if (encmode.equals("rsa")) {
+ ciphertext = JsonPath.read(results.toString(), "$.ciphertext").toString();
+ returnvalue = ciphertext;
+
+ } else {
+ ciphertext = JsonPath.read(results.toString(), "$.ciphertext").toString();
+ String tagtext = JsonPath.read(results.toString(), "$.tag").toString();
+ returnvalue = filetagname + tagtext + filetagsep + ciphertext;
+
+ }
+ // System.out.println("cipher text = " + ciphertext);
+ } catch (Exception e) {
+ // TODO: handle exception
+ System.out.println(e.getMessage());
+
+ String code = JsonPath.read(results.toString(), "$.code").toString();
+ String msg = JsonPath.read(results.toString(), "$.codeDesc").toString();
+ System.out.println("code " + code);
+ System.out.println("code desc " + msg);
+ StackTraceElement[] ste = e.getStackTrace();
+ for (int j = 0; j < ste.length; j++) {
+ System.out.println(ste[j]);
+
+ }
+
+ System.exit(-1);
+ }
+
+ return returnvalue;
+
+ }
+
+ private static boolean isNumeric(String str) {
+ for (char c : str.toCharArray()) {
+
+ if (!Character.isDigit(c)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static boolean isAlpha(String str) {
+
+ for (char c : str.toCharArray()) {
+ if (!Character.isAlphabetic(c)) {
+ // if (!Character.isLetter(c)) {
+ return false;
+ }
+ }
+ return true;
+
+ }
+
+ private static String getSCUnique(String name) {
+ StringBuffer returnvalue = new StringBuffer();
+ HashMap hm = new HashMap();
+ String specialCharacters = " !#$%&'()*+,-./:;<=>?@[]^_`{|}~";
+ String str2[] = name.split("");
+ int count = 0;
+ for (int i = 0; i < str2.length; i++) {
+ if (specialCharacters.contains(str2[i])) {
+ count++;
+ hm.put(str2[i], str2[i]);
+ }
+ }
+
+ Set set = hm.entrySet();
+ Iterator i = set.iterator();
+
+ // Display elements
+ while (i.hasNext()) {
+ Map.Entry me = (Map.Entry) i.next();
+ returnvalue.append(me.getKey());
+ }
+
+ return returnvalue.toString();
+ }
+
+ private static final String ALPHA_NUMERIC_STRING = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+ private static final String ALPHA = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+
+ private static String randomAlpha(int count) {
+ StringBuilder builder = new StringBuilder();
+ while (count-- != 0) {
+ int character = (int) (Math.random() * ALPHA.length());
+ builder.append(ALPHA.charAt(character));
+ }
+ return builder.toString();
+ }
+
+ private static String randomAlphaNumeric(int count) {
+ StringBuilder builder = new StringBuilder();
+ while (count-- != 0) {
+ int character = (int) (Math.random() * ALPHA_NUMERIC_STRING.length());
+ builder.append(ALPHA_NUMERIC_STRING.charAt(character));
+ }
+ return builder.toString();
+ }
+
+ private static final String NUMERIC_STRING = "0123456789";
+
+ private static String randomNumeric(int count) {
+ StringBuilder builder = new StringBuilder();
+ while (count-- != 0) {
+ int character = (int) (Math.random() * NUMERIC_STRING.length());
+ builder.append(NUMERIC_STRING.charAt(character));
+ }
+ return builder.toString();
+ }
+
+}
diff --git a/rest/src/main/java/com/thales/cm/rest/cmrestdemo/CMRESTJsonDemo.jar b/rest/cmrest/src/main/java/com/thales/cm/rest/cmrestdemo/CMRESTJsonDemo.jar
similarity index 100%
rename from rest/src/main/java/com/thales/cm/rest/cmrestdemo/CMRESTJsonDemo.jar
rename to rest/cmrest/src/main/java/com/thales/cm/rest/cmrestdemo/CMRESTJsonDemo.jar
diff --git a/rest/src/main/java/com/thales/cm/rest/cmrestdemo/CMRESTJsonDemo.java b/rest/cmrest/src/main/java/com/thales/cm/rest/cmrestdemo/CMRESTJsonDemo.java
similarity index 96%
rename from rest/src/main/java/com/thales/cm/rest/cmrestdemo/CMRESTJsonDemo.java
rename to rest/cmrest/src/main/java/com/thales/cm/rest/cmrestdemo/CMRESTJsonDemo.java
index 8c6cc0c3..7888683c 100644
--- a/rest/src/main/java/com/thales/cm/rest/cmrestdemo/CMRESTJsonDemo.java
+++ b/rest/cmrest/src/main/java/com/thales/cm/rest/cmrestdemo/CMRESTJsonDemo.java
@@ -1,467 +1,467 @@
-package com.thales.cm.rest.cmrestdemo;
-
-import java.util.Map;
-/** This application demonstrates how to use the CM Rest API to encrypt a particular json element (message) in a file.
- * it leverages a helper class located at:
- * https://github.com/thalescpl-io/CipherTrust_Application_Protection/tree/master/rest/src/main/java/com/thales/cm/rest/cmhelper
- * Format for input file is:
- * "msgcontent": {
- "recordnbr": 11,
- "title": "Your title",
- "message": "NRajFQ4P4EvuSe27VwG4NWQV5rb0FzO6upYSRp1bM4qKvPhBiJmc5BtEv42W6VjMMTMIvvN4BBCQ9JkiwJxeWL7kdY1",
- "username": "Sam Smith"
- }
- * To run this example need to set the following environment variables.
- *
- * set cmdebug=0
-SET cmuserid=admin
-echo %cmuserid%
-set cmpassword=yourpwd
-set cmserver=yourcmipaddress
-set cmkey=yourencryptionkey (must be in root domain of CM)
-set cmdataformat=alphanumeric (keep this as alphanumeric)
-set cmfile=C:\\jars\\CMRestDemo\\messages.json
-java -jar CMRESTJsonDemo.jar
-
- */
-import java.util.Scanner;
-
-import com.thales.cm.rest.cmhelper.CipherTrustManagerHelper;
-
-import net.minidev.json.JSONArray;
-import net.minidev.json.JSONObject;
-import net.minidev.json.parser.JSONParser;
-import net.minidev.json.parser.ParseException;
-
-import java.io.*;
-
-public class CMRESTJsonDemo {
-
- String fname = "messages.json";
-
- public static void main(String[] input) throws Exception {
- String datainput = "1";
- CMRESTJsonDemo dcng = new CMRESTJsonDemo();
- dcng.fname = dcng.getFile();
- while (true) {
-
- datainput = dcng.displayMenu();
-
- switch (datainput) {
- case "1":
- dcng.readJSONFile();
- break;
- case "2":
- dcng.addRecord();
- break;
- case "3":
- String found = dcng.viewRecord();
- if (found.equals("false"))
- System.out.println("Record not found.....");
- break;
- case "4":
- System.exit(1);
- break;
- default:
- dcng.readJSONFile();
- break;
- }
-
- }
-
- }
- /**
- * Gets the input file.
- *
- *
- * @return string file name from environment variable called cmfile
- */
- private String getFile() {
-
- String cmfile = null;
- Map env = System.getenv();
- for (String envName : env.keySet()) {
- if (envName.equalsIgnoreCase("cmfile"))
- {
- cmfile = env.get(envName);
- }
- }
- if (cmfile != null)
- fname = cmfile;
- else cmfile = fname;
- return cmfile;
-
- }
-
- /**
- * Adds a record to the input file.
- *
- */
-
- private void addRecord() throws Exception {
- String username = "";
- String title = "1";
- String message = "";
- int totalrecords = 0;
-
- CipherTrustManagerHelper cmrest = new CipherTrustManagerHelper();
- String tkn = cmrest.getToken();
-
- JSONParser jsonParser = new JSONParser();
- JSONArray messageListNew = new JSONArray();
- try (FileReader reader = new FileReader(fname)) {
- // Read JSON file
- Object obj = jsonParser.parse(reader);
-
- JSONArray messageList = (JSONArray) obj;
- // System.out.println(messageList);
-
- for (int i = 0; i < messageList.size(); i++) {
- totalrecords++;
-
- JSONObject explrObject = (JSONObject) messageList.get(i);
- messageListNew.add(explrObject);
-
- }
-
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- } catch (ParseException e) {
- e.printStackTrace();
- }
-
- Scanner scan = new Scanner(System.in);
-
- System.out.print("Enter Customer Name : ");
- username = scan.nextLine();
-
- System.out.print("Enter Title : ");
- title = scan.nextLine();
-
- System.out.print("Enter Message : ");
- message = scan.nextLine();
-
- // First Record
- JSONObject messageDetails = new JSONObject();
- messageDetails.put("recordnbr", totalrecords + 1);
- messageDetails.put("username", username);
- messageDetails.put("title", title);
-
- // This is the call to encrypt the data
- String results = cmrest.cmRESTProtect("fpe", message, "encrypt");
- messageDetails.put("message", results);
-
- JSONObject messageObject = new JSONObject();
- messageObject.put("msgcontent", messageDetails);
- messageListNew.add(messageObject);
- System.out.println("added new record ");
- parseMessageObject(messageObject);
- // Write JSON file
- try (FileWriter file = new FileWriter(fname)) {
- file.write(messageListNew.toJSONString());
- file.flush();
- file.close();
-
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- //scan.close();
- }
-
- /**
- * Views a record in the inputfile.
- *
- *
- * @return string returns "true" if found "false if not found
- */
-
- private String viewRecord() throws Exception {
-
- String found = "true";
- String testing = "0";
- CipherTrustManagerHelper cmrest = new CipherTrustManagerHelper();
-
- Scanner scan = new Scanner(System.in);
- String username;
- String password;
- String recordnbr;
- System.out.print("Enter CM UserName : ");
- username = scan.nextLine();
- System.out.print("Enter CM Password : ");
- if (testing.equals("1")) {
- password = scan.nextLine();
- } else {
- Console console = System.console();
- if (console == null) {
- System.out.println("No console: not in interactive mode!");
- System.exit(0);
- }
- char[] passwordchar = console.readPassword();
- password = new String(passwordchar);
-
- }
- cmrest.username = username;
- cmrest.password = password;
-
- System.out.print("Enter Record Number to view : ");
- recordnbr = scan.nextLine();
- String tkn = cmrest.getToken();
-
- boolean validnbr = this.isNumeric(recordnbr);
- if (!validnbr)
- {
- System.out.println("Not a number please enter a valid record number ");
- while (!validnbr)
- {
- recordnbr = scan.nextLine();
- validnbr = this.isNumeric(recordnbr);
- if (!validnbr)
- {
- System.out.println("Not a number please enter a valid record number ");
- }
-
- }
-
- }
- Integer recordnbrInt = new Integer(recordnbr);
- int recordnbrint = recordnbrInt.intValue();
-
- JSONObject messageObject = this.findRecord(recordnbrint);
-
- if (messageObject != null) {
- this.parseMessageObjectDecrypt(messageObject, cmrest);
-
- } else
- found = "false";
-
- //scan.close();
- return found;
-
- }
-
- private boolean isNumeric(String strNum) {
- if (strNum == null) {
- return false;
- }
- try {
- double d = Double.parseDouble(strNum);
- } catch (NumberFormatException nfe) {
- return false;
- }
- return true;
- }
-
- /**
- * Displays the menu
- *
- *
- * @return string Value chosen by user
- */
-
- private String displayMenu() {
-
- System.out.println(" ");
- System.out.println("v1.2");
- System.out.println("---------------------------------------");
- System.out.println("CipherTrust Manager REST JSON file Demo");
- System.out.println("---------------------------------------");
- System.out.println("Menu");
- System.out.println("1. Print file " + fname);
- System.out.println("2. Add a Record");
- System.out.println("3. View a Record");
- System.out.println("4. Exit");
-
- String datainput = "1";
- Scanner scan = new Scanner(System.in);
- /* enter filename with extension to open and read its content */
-
- System.out.print("Enter number from above : ");
- datainput = scan.nextLine();
-
- //scan.close();
- return datainput;
-
- }
-
- /**
- * Finds the record number passed in.
- *
- *
- * @param recordnumber
- * recordnumber to search
- * @return JSONObject JSON Element of found record or null if not found
- */
- private JSONObject findRecord(int recordnbr) {
-
- JSONParser jsonParser = new JSONParser();
- JSONObject explrObject = null;
- try (FileReader reader = new FileReader(fname)) {
- // Read JSON file
- Object obj = jsonParser.parse(reader);
-
- JSONArray messageList = (JSONArray) obj;
- // System.out.println(messageList);
- System.out.println("Total Number of Records " + messageList.size());
-
- if(recordnbr > messageList.size() || recordnbr <=0)
- return null;
-
- System.out.println(" ");
- for (int i = 0; i < messageList.size(); i++) {
-
- // store each object in JSONObject
- explrObject = (JSONObject) messageList.get(i);
- JSONObject messageObject = (JSONObject) explrObject.get("msgcontent");
-
- int recordnbrint = (int) messageObject.get("recordnbr");
- if (recordnbr == recordnbrint)
- break;
-
- }
-
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- } catch (ParseException e) {
- e.printStackTrace();
- }
-
- return explrObject;
-
- }
-
- /**
- * Reads all records in a file
- *
- */
-
- private void readJSONFile() {
-
- JSONParser jsonParser = new JSONParser();
-
- try (FileReader reader = new FileReader(fname)) {
- // Read JSON file
- Object obj = jsonParser.parse(reader);
-
- JSONArray messageList = (JSONArray) obj;
-
- System.out.println(" ");
- System.out.println("Total Number of Records " + messageList.size());
- if (messageList.size() == 0)
- System.out.println(" No records to list please add a record");
- System.out.println(" ");
- System.out.println("---------------------------------------");
-
- for (int i = 0; i < messageList.size(); i++) {
-
- // store each object in JSONObject
- JSONObject explrObject = (JSONObject) messageList.get(i);
- parseMessageObject(explrObject);
- // System.out.println(explrObject.get("msgcontent"));
- }
- System.out.println("---------------------------------------");
- System.out.println(" ");
-
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- } catch (ParseException e) {
- e.printStackTrace();
- }
-
- }
-
- /**
- * Prints the json object passed in. First checks to see if the user has
- * access to the key by
- *
- *
- * @param JSONObject
- * JSON Object to print
- * @param CipherTrustManagerHelper
- * Helper class to make calls to CM
- */
- private void parseMessageObjectDecrypt(JSONObject message, CipherTrustManagerHelper cmrest)
- throws IOException {
-
- // First call CM to see if the user has access to the key.
- int keysize = cmrest.getKeySize();
- if (keysize == 0) {
- System.out.println(" ");
- System.out.println("No access to key!!!!!!!!!!!!!");
- System.out.println(" ");
- }
- JSONObject messageObject = (JSONObject) message.get("msgcontent");
-
- int recordnbr = (int) messageObject.get("recordnbr");
- String username = (String) messageObject.get("username");
- String title = (String) messageObject.get("title");
- String message1 = (String) messageObject.get("message");
- String results = null;
- try {
- if (keysize > 0)
- // This is the call to decrypt the data
- results = cmrest.cmRESTProtect("fpe", message1, "decrypt");
- else
- results = message1;
-
- } catch (Exception e) {
- if (e.getMessage().contains("Resource"))
- System.out.println("No access to key");
-
- e.printStackTrace();
-
- }
- System.out.println("" + recordnbr + "," + username + "," + title + "," + results);
- }
-
- /**
- * Prints the json object passed in.
- *
- *
- * @param JSONObject
- * JSON Object to print
- */
-
- private void parseMessageObject(JSONObject message) {
- // Get msgcontent object within list
- JSONObject messageObject = (JSONObject) message.get("msgcontent");
-
- int recordnbr = (int) messageObject.get("recordnbr");
- String username = (String) messageObject.get("username");
- String title = (String) messageObject.get("title");
- String message1 = (String) messageObject.get("message");
- System.out.println("" + recordnbr + "," + username + "," + title + "," + message1);
- }
-
- // Currently not used.
-
- private void printRecords() {
- String line = null;
- try {
- /* FileReader reads text files in the default encoding */
- FileReader fileReader = new FileReader(fname);
-
- /* always wrap the FileReader in BufferedReader */
- BufferedReader bufferedReader = new BufferedReader(fileReader);
-
- while ((line = bufferedReader.readLine()) != null) {
- System.out.println(line);
- }
-
- /* always close the file after use */
- bufferedReader.close();
- System.out.println("-------------------");
- System.out.println(" ");
- System.out.println(" ");
- } catch (IOException ex) {
- System.out.println("Error reading file named '" + fname + "'");
- }
-
- }
-}
+package com.thales.cm.rest.cmrestdemo;
+
+import java.util.Map;
+/** This application demonstrates how to use the CM Rest API to encrypt a particular json element (message) in a file.
+ * it leverages a helper class located at:
+ * https://github.com/thalescpl-io/CipherTrust_Application_Protection/tree/master/rest/src/main/java/com/thales/cm/rest/cmhelper
+ * Format for input file is:
+ * "msgcontent": {
+ "recordnbr": 11,
+ "title": "Your title",
+ "message": "NRajFQ4P4EvuSe27VwG4NWQV5rb0FzO6upYSRp1bM4qKvPhBiJmc5BtEv42W6VjMMTMIvvN4BBCQ9JkiwJxeWL7kdY1",
+ "username": "Sam Smith"
+ }
+ * To run this example need to set the following environment variables.
+ *
+ * set cmdebug=0
+SET cmuserid=admin
+echo %cmuserid%
+set cmpassword=yourpwd
+set cmserver=yourcmipaddress
+set cmkey=yourencryptionkey (must be in root domain of CM)
+set cmdataformat=alphanumeric (keep this as alphanumeric)
+set cmfile=C:\\jars\\CMRestDemo\\messages.json
+java -jar CMRESTJsonDemo.jar
+
+ */
+import java.util.Scanner;
+
+import com.thales.cm.rest.cmhelper.CipherTrustManagerHelper;
+
+import net.minidev.json.JSONArray;
+import net.minidev.json.JSONObject;
+import net.minidev.json.parser.JSONParser;
+import net.minidev.json.parser.ParseException;
+
+import java.io.*;
+
+public class CMRESTJsonDemo {
+
+ String fname = "messages.json";
+
+ public static void main(String[] input) throws Exception {
+ String datainput = "1";
+ CMRESTJsonDemo dcng = new CMRESTJsonDemo();
+ dcng.fname = dcng.getFile();
+ while (true) {
+
+ datainput = dcng.displayMenu();
+
+ switch (datainput) {
+ case "1":
+ dcng.readJSONFile();
+ break;
+ case "2":
+ dcng.addRecord();
+ break;
+ case "3":
+ String found = dcng.viewRecord();
+ if (found.equals("false"))
+ System.out.println("Record not found.....");
+ break;
+ case "4":
+ System.exit(1);
+ break;
+ default:
+ dcng.readJSONFile();
+ break;
+ }
+
+ }
+
+ }
+ /**
+ * Gets the input file.
+ *
+ *
+ * @return string file name from environment variable called cmfile
+ */
+ private String getFile() {
+
+ String cmfile = null;
+ Map env = System.getenv();
+ for (String envName : env.keySet()) {
+ if (envName.equalsIgnoreCase("cmfile"))
+ {
+ cmfile = env.get(envName);
+ }
+ }
+ if (cmfile != null)
+ fname = cmfile;
+ else cmfile = fname;
+ return cmfile;
+
+ }
+
+ /**
+ * Adds a record to the input file.
+ *
+ */
+
+ private void addRecord() throws Exception {
+ String username = "";
+ String title = "1";
+ String message = "";
+ int totalrecords = 0;
+
+ CipherTrustManagerHelper cmrest = new CipherTrustManagerHelper();
+ String tkn = cmrest.getToken();
+
+ JSONParser jsonParser = new JSONParser();
+ JSONArray messageListNew = new JSONArray();
+ try (FileReader reader = new FileReader(fname)) {
+ // Read JSON file
+ Object obj = jsonParser.parse(reader);
+
+ JSONArray messageList = (JSONArray) obj;
+ // System.out.println(messageList);
+
+ for (int i = 0; i < messageList.size(); i++) {
+ totalrecords++;
+
+ JSONObject explrObject = (JSONObject) messageList.get(i);
+ messageListNew.add(explrObject);
+
+ }
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (ParseException e) {
+ e.printStackTrace();
+ }
+
+ Scanner scan = new Scanner(System.in);
+
+ System.out.print("Enter Customer Name : ");
+ username = scan.nextLine();
+
+ System.out.print("Enter Title : ");
+ title = scan.nextLine();
+
+ System.out.print("Enter Message : ");
+ message = scan.nextLine();
+
+ // First Record
+ JSONObject messageDetails = new JSONObject();
+ messageDetails.put("recordnbr", totalrecords + 1);
+ messageDetails.put("username", username);
+ messageDetails.put("title", title);
+
+ // This is the call to encrypt the data
+ String results = cmrest.cmRESTProtect("fpe", message, "encrypt");
+ messageDetails.put("message", results);
+
+ JSONObject messageObject = new JSONObject();
+ messageObject.put("msgcontent", messageDetails);
+ messageListNew.add(messageObject);
+ System.out.println("added new record ");
+ parseMessageObject(messageObject);
+ // Write JSON file
+ try (FileWriter file = new FileWriter(fname)) {
+ file.write(messageListNew.toJSONString());
+ file.flush();
+ file.close();
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ //scan.close();
+ }
+
+ /**
+ * Views a record in the inputfile.
+ *
+ *
+ * @return string returns "true" if found "false if not found
+ */
+
+ private String viewRecord() throws Exception {
+
+ String found = "true";
+ String testing = "0";
+ CipherTrustManagerHelper cmrest = new CipherTrustManagerHelper();
+
+ Scanner scan = new Scanner(System.in);
+ String username;
+ String password;
+ String recordnbr;
+ System.out.print("Enter CM UserName : ");
+ username = scan.nextLine();
+ System.out.print("Enter CM Password : ");
+ if (testing.equals("1")) {
+ password = scan.nextLine();
+ } else {
+ Console console = System.console();
+ if (console == null) {
+ System.out.println("No console: not in interactive mode!");
+ System.exit(0);
+ }
+ char[] passwordchar = console.readPassword();
+ password = new String(passwordchar);
+
+ }
+ cmrest.username = username;
+ cmrest.password = password;
+
+ System.out.print("Enter Record Number to view : ");
+ recordnbr = scan.nextLine();
+ String tkn = cmrest.getToken();
+
+ boolean validnbr = this.isNumeric(recordnbr);
+ if (!validnbr)
+ {
+ System.out.println("Not a number please enter a valid record number ");
+ while (!validnbr)
+ {
+ recordnbr = scan.nextLine();
+ validnbr = this.isNumeric(recordnbr);
+ if (!validnbr)
+ {
+ System.out.println("Not a number please enter a valid record number ");
+ }
+
+ }
+
+ }
+ Integer recordnbrInt = new Integer(recordnbr);
+ int recordnbrint = recordnbrInt.intValue();
+
+ JSONObject messageObject = this.findRecord(recordnbrint);
+
+ if (messageObject != null) {
+ this.parseMessageObjectDecrypt(messageObject, cmrest);
+
+ } else
+ found = "false";
+
+ //scan.close();
+ return found;
+
+ }
+
+ private boolean isNumeric(String strNum) {
+ if (strNum == null) {
+ return false;
+ }
+ try {
+ double d = Double.parseDouble(strNum);
+ } catch (NumberFormatException nfe) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Displays the menu
+ *
+ *
+ * @return string Value chosen by user
+ */
+
+ private String displayMenu() {
+
+ System.out.println(" ");
+ System.out.println("v1.2");
+ System.out.println("---------------------------------------");
+ System.out.println("CipherTrust Manager REST JSON file Demo");
+ System.out.println("---------------------------------------");
+ System.out.println("Menu");
+ System.out.println("1. Print file " + fname);
+ System.out.println("2. Add a Record");
+ System.out.println("3. View a Record");
+ System.out.println("4. Exit");
+
+ String datainput = "1";
+ Scanner scan = new Scanner(System.in);
+ /* enter filename with extension to open and read its content */
+
+ System.out.print("Enter number from above : ");
+ datainput = scan.nextLine();
+
+ //scan.close();
+ return datainput;
+
+ }
+
+ /**
+ * Finds the record number passed in.
+ *
+ *
+ * @param recordnumber
+ * recordnumber to search
+ * @return JSONObject JSON Element of found record or null if not found
+ */
+ private JSONObject findRecord(int recordnbr) {
+
+ JSONParser jsonParser = new JSONParser();
+ JSONObject explrObject = null;
+ try (FileReader reader = new FileReader(fname)) {
+ // Read JSON file
+ Object obj = jsonParser.parse(reader);
+
+ JSONArray messageList = (JSONArray) obj;
+ // System.out.println(messageList);
+ System.out.println("Total Number of Records " + messageList.size());
+
+ if(recordnbr > messageList.size() || recordnbr <=0)
+ return null;
+
+ System.out.println(" ");
+ for (int i = 0; i < messageList.size(); i++) {
+
+ // store each object in JSONObject
+ explrObject = (JSONObject) messageList.get(i);
+ JSONObject messageObject = (JSONObject) explrObject.get("msgcontent");
+
+ int recordnbrint = (int) messageObject.get("recordnbr");
+ if (recordnbr == recordnbrint)
+ break;
+
+ }
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (ParseException e) {
+ e.printStackTrace();
+ }
+
+ return explrObject;
+
+ }
+
+ /**
+ * Reads all records in a file
+ *
+ */
+
+ private void readJSONFile() {
+
+ JSONParser jsonParser = new JSONParser();
+
+ try (FileReader reader = new FileReader(fname)) {
+ // Read JSON file
+ Object obj = jsonParser.parse(reader);
+
+ JSONArray messageList = (JSONArray) obj;
+
+ System.out.println(" ");
+ System.out.println("Total Number of Records " + messageList.size());
+ if (messageList.size() == 0)
+ System.out.println(" No records to list please add a record");
+ System.out.println(" ");
+ System.out.println("---------------------------------------");
+
+ for (int i = 0; i < messageList.size(); i++) {
+
+ // store each object in JSONObject
+ JSONObject explrObject = (JSONObject) messageList.get(i);
+ parseMessageObject(explrObject);
+ // System.out.println(explrObject.get("msgcontent"));
+ }
+ System.out.println("---------------------------------------");
+ System.out.println(" ");
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (ParseException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+ /**
+ * Prints the json object passed in. First checks to see if the user has
+ * access to the key by
+ *
+ *
+ * @param JSONObject
+ * JSON Object to print
+ * @param CipherTrustManagerHelper
+ * Helper class to make calls to CM
+ */
+ private void parseMessageObjectDecrypt(JSONObject message, CipherTrustManagerHelper cmrest)
+ throws IOException {
+
+ // First call CM to see if the user has access to the key.
+ int keysize = cmrest.getKeySize();
+ if (keysize == 0) {
+ System.out.println(" ");
+ System.out.println("No access to key!!!!!!!!!!!!!");
+ System.out.println(" ");
+ }
+ JSONObject messageObject = (JSONObject) message.get("msgcontent");
+
+ int recordnbr = (int) messageObject.get("recordnbr");
+ String username = (String) messageObject.get("username");
+ String title = (String) messageObject.get("title");
+ String message1 = (String) messageObject.get("message");
+ String results = null;
+ try {
+ if (keysize > 0)
+ // This is the call to decrypt the data
+ results = cmrest.cmRESTProtect("fpe", message1, "decrypt");
+ else
+ results = message1;
+
+ } catch (Exception e) {
+ if (e.getMessage().contains("Resource"))
+ System.out.println("No access to key");
+
+ e.printStackTrace();
+
+ }
+ System.out.println("" + recordnbr + "," + username + "," + title + "," + results);
+ }
+
+ /**
+ * Prints the json object passed in.
+ *
+ *
+ * @param JSONObject
+ * JSON Object to print
+ */
+
+ private void parseMessageObject(JSONObject message) {
+ // Get msgcontent object within list
+ JSONObject messageObject = (JSONObject) message.get("msgcontent");
+
+ int recordnbr = (int) messageObject.get("recordnbr");
+ String username = (String) messageObject.get("username");
+ String title = (String) messageObject.get("title");
+ String message1 = (String) messageObject.get("message");
+ System.out.println("" + recordnbr + "," + username + "," + title + "," + message1);
+ }
+
+ // Currently not used.
+
+ private void printRecords() {
+ String line = null;
+ try {
+ /* FileReader reads text files in the default encoding */
+ FileReader fileReader = new FileReader(fname);
+
+ /* always wrap the FileReader in BufferedReader */
+ BufferedReader bufferedReader = new BufferedReader(fileReader);
+
+ while ((line = bufferedReader.readLine()) != null) {
+ System.out.println(line);
+ }
+
+ /* always close the file after use */
+ bufferedReader.close();
+ System.out.println("-------------------");
+ System.out.println(" ");
+ System.out.println(" ");
+ } catch (IOException ex) {
+ System.out.println("Error reading file named '" + fname + "'");
+ }
+
+ }
+}
diff --git a/rest/src/main/java/com/thales/cm/rest/cmrestdemo/README.md b/rest/cmrest/src/main/java/com/thales/cm/rest/cmrestdemo/README.md
similarity index 97%
rename from rest/src/main/java/com/thales/cm/rest/cmrestdemo/README.md
rename to rest/cmrest/src/main/java/com/thales/cm/rest/cmrestdemo/README.md
index 9f105570..45a37f08 100644
--- a/rest/src/main/java/com/thales/cm/rest/cmrestdemo/README.md
+++ b/rest/cmrest/src/main/java/com/thales/cm/rest/cmrestdemo/README.md
@@ -1,6 +1,6 @@
-# REST/Sample Code
-
-Integrations or Sample Code for applications using CipherTrust Manager Application Encryption & Decryption
-
-## Integrations
-CMRESTJsonDemo.java - encrypts and decrypts a field in a json file. Uses Format Preserve Encryption.
+# REST/Sample Code
+
+Integrations or Sample Code for applications using CipherTrust Manager Application Encryption & Decryption
+
+## Integrations
+CMRESTJsonDemo.java - encrypts and decrypts a field in a json file. Uses Format Preserve Encryption.
diff --git a/rest/src/main/java/com/thales/cm/rest/cmrestdemo/messages.json b/rest/cmrest/src/main/java/com/thales/cm/rest/cmrestdemo/messages.json
similarity index 100%
rename from rest/src/main/java/com/thales/cm/rest/cmrestdemo/messages.json
rename to rest/cmrest/src/main/java/com/thales/cm/rest/cmrestdemo/messages.json
diff --git a/rest/src/main/java/com/thales/cm/rest/cmrestdemo/run-cmrestjsonwin.bat b/rest/cmrest/src/main/java/com/thales/cm/rest/cmrestdemo/run-cmrestjsonwin.bat
similarity index 96%
rename from rest/src/main/java/com/thales/cm/rest/cmrestdemo/run-cmrestjsonwin.bat
rename to rest/cmrest/src/main/java/com/thales/cm/rest/cmrestdemo/run-cmrestjsonwin.bat
index 853d30a4..924ab4d7 100644
--- a/rest/src/main/java/com/thales/cm/rest/cmrestdemo/run-cmrestjsonwin.bat
+++ b/rest/cmrest/src/main/java/com/thales/cm/rest/cmrestdemo/run-cmrestjsonwin.bat
@@ -1,8 +1,8 @@
-set cmdebug=0
-SET cmuserid=admin
-echo %cmuserid%
-set cmpassword=yourpwd
-set cmserver=yourcmserver
-set cmkey=yourcmkeyAES256
-set cmdataformat=alphanumeric
+set cmdebug=0
+SET cmuserid=admin
+echo %cmuserid%
+set cmpassword=yourpwd
+set cmserver=yourcmserver
+set cmkey=yourcmkeyAES256
+set cmdataformat=alphanumeric
java -jar CMRESTJsonDemo.jar
\ No newline at end of file
diff --git a/rest/src/main/java/com/thales/cm/rest/cmrestdemo/run-cmrrestjsonlinux.sh b/rest/cmrest/src/main/java/com/thales/cm/rest/cmrestdemo/run-cmrrestjsonlinux.sh
similarity index 100%
rename from rest/src/main/java/com/thales/cm/rest/cmrestdemo/run-cmrrestjsonlinux.sh
rename to rest/cmrest/src/main/java/com/thales/cm/rest/cmrestdemo/run-cmrrestjsonlinux.sh
diff --git a/rest/crdp/documentation/ThalesCRDP-CMSetup-Tutorial.pdf b/rest/crdp/documentation/ThalesCRDP-CMSetup-Tutorial.pdf
new file mode 100644
index 00000000..32bd071e
Binary files /dev/null and b/rest/crdp/documentation/ThalesCRDP-CMSetup-Tutorial.pdf differ
diff --git a/rest/crdp/java/pom.xml b/rest/crdp/java/pom.xml
new file mode 100644
index 00000000..1003a66e
--- /dev/null
+++ b/rest/crdp/java/pom.xml
@@ -0,0 +1,22 @@
+
+ 4.0.0
+ Thales
+ Thales-CRDP
+ 0.0.1-SNAPSHOT
+
+ 32.0.0-jre
+ 2.9.0
+
+
+
+ com.google.code.gson
+ gson
+ ${gson.version}
+
+
+ com.squareup.okhttp3
+ okhttp
+ 4.10.0
+
+
+
\ No newline at end of file
diff --git a/rest/crdp/java/src/main/java/com/example/CRDPBulkOperation.java b/rest/crdp/java/src/main/java/com/example/CRDPBulkOperation.java
new file mode 100644
index 00000000..1dee019f
--- /dev/null
+++ b/rest/crdp/java/src/main/java/com/example/CRDPBulkOperation.java
@@ -0,0 +1,335 @@
+package com.example;
+
+/**
+* Sample code is provided for educational purposes.
+* No warranty of any kind, either expressed or implied by fact or law.
+* Use of this item is not restricted by copyright or license terms.
+*/
+// Standard JCE classes.
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+
+/**
+ * This sample shows how to use the protect/reveal REST API using both the regular protect and the protectbulk.
+ *
+ */
+public class CRDPBulkOperation {
+
+ private static int BATCHLIMIT = 5000;
+
+ public static void main(String[] args) throws Exception {
+
+ String crdp_container_IP = "192.168.159.143";
+ int batchsize = 2;
+ String protectURL = "http://" + crdp_container_IP + ":8090/v1/protect";
+ boolean debug = false;
+
+ String protection_profile = "plain-nbr-internal";
+ String modeofoperation = "regular";
+
+ if (!args[0].contains("null")) {
+ debug = args[0].equalsIgnoreCase("true");
+ }
+
+ if (!args[1].contains("null")) {
+ modeofoperation = args[1];
+ }
+ if (modeofoperation.equalsIgnoreCase("bulk"))
+ protectURL = protectURL + "bulk";
+
+ int index = 0;
+ int totalrecords = 0;
+ int recordcnter = 0;
+ long startTime = System.currentTimeMillis();
+ int nbrofchunks = 0;
+
+ //emp-id1k.txt, emp-id5k.txt,emp-nbr-smallout.txt
+ String filePath = "C:\\data\\emp-nbr-smallout.txt"; // Change
+
+
+
+ if (modeofoperation.equalsIgnoreCase("regular")) {
+ totalrecords = protectFromFile(protectURL, debug, filePath, protection_profile);
+ nbrofchunks = 1;
+ } else {
+ List strings = readStringsFromFile(filePath);
+ int totalnumberofrecords = strings.size();
+ int recordsleft = totalnumberofrecords;
+
+ List batchnnn = new ArrayList<>();
+ startTime = System.currentTimeMillis();
+
+ if (batchsize > totalnumberofrecords)
+ batchsize = totalnumberofrecords;
+ if (batchsize >= BATCHLIMIT)
+ batchsize = BATCHLIMIT;
+
+ while (index < totalnumberofrecords) {
+
+ for (int i = 0; i < batchsize && index < totalnumberofrecords; i++) {
+ batchnnn.add(strings.get(recordcnter));
+ recordcnter++;
+ index++;
+ recordsleft--;
+
+ }
+
+ totalrecords = totalrecords + protectFromFileBulk(protectURL, debug, batchnnn, batchsize, index,
+ totalnumberofrecords, recordsleft, protection_profile);
+ batchnnn = new ArrayList<>();
+ nbrofchunks++;
+
+ }
+ }
+
+ long endTime = System.currentTimeMillis();
+ long totalTime = endTime - startTime;
+ System.out.println("Time taken CDRP test: " + totalTime + " milliseconds");
+ if (modeofoperation.equalsIgnoreCase("bulk")) {
+ System.out.println("BatchSize: " + batchsize);
+ System.out.println("Nbrofchunks : " + nbrofchunks);
+ }
+ System.out.println("Total records = " + totalrecords);
+
+ }
+
+ public static int protectFromFileBulk(String url, boolean debug, List strings, int batchsize, int index,
+ int totalnumberofrecords, int recordsleft, String protection_profile) throws Exception {
+
+ JsonObject crdp_payload = new JsonObject();
+ String jsonBody = null;
+ JsonArray crdp_payload_array = new JsonArray();
+ String inputdataarray = null;
+ StringBuffer protection_policy_buff = new StringBuffer();
+
+ int totalnbrofrecords = 0;
+
+ String line = "";
+ int count = 0;
+ int stopper = batchsize;
+ if (index == totalnumberofrecords)
+ stopper = recordsleft + 1;
+
+ String formattedElement = String.format("\"protection_policy_name\" : \"%s\"", protection_profile);
+ protection_policy_buff.append(formattedElement);
+ protection_policy_buff.append(",");
+
+ // Loop is only needed if you have different policies for each data value in the array.
+ // Example:
+ /*
+ * {"protection_policy_name": "plain-nbr-internal","protection_policy_name": "plain-alpha-internal",
+ * "data_array": [ "1234567812345678", "abcdef" ]}
+ *
+ * for (int i = 0; i < stopper; i++) {
+ *
+ * String formattedElement = String.format("\"protection_policy_name\" : \"%s\"", protection_profile);
+ * protection_policy_buff.append(formattedElement); protection_policy_buff.append(",");
+ *
+ * }
+ */
+
+ for (String str : strings) {
+ totalnbrofrecords++;
+
+ crdp_payload_array.add(str);
+ count++;
+
+ if (count == batchsize) {
+
+ crdp_payload.add("data_array", crdp_payload_array);
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ inputdataarray = protection_policy_buff.toString();
+ jsonBody = inputdataarray.replace("{", " ");
+
+ jsonBody = "{" + jsonBody;
+
+ if (debug)
+ System.out.println("json body ---------------" + jsonBody);
+
+ makeCMCall(jsonBody, url, debug);
+
+ formattedElement = String.format("\"protection_policy_name\" : \"%s\"", protection_profile);
+ protection_policy_buff.append(formattedElement);
+ protection_policy_buff.append(",");
+ // Loop is only needed if you have different policies for each data value in the array.
+ /*
+ * for (int i = 0; i < stopper; i++) { String formattedElement =
+ * String.format("\"protection_policy_name\" : \"%s\"", protection_profile);
+ * protection_policy_buff.append(formattedElement); protection_policy_buff.append(",");
+ *
+ * }
+ */
+ count = 0;
+ }
+ line = str;
+ }
+
+ if (count > 0) {
+ crdp_payload.add("data_array", crdp_payload_array);
+ inputdataarray = crdp_payload.toString();
+ protection_policy_buff.append(inputdataarray);
+ inputdataarray = protection_policy_buff.toString();
+ jsonBody = inputdataarray.replace("{", " ");
+
+ jsonBody = "{" + jsonBody;
+
+ if (debug) {
+ System.out.println(jsonBody.toString());
+ System.out.println("count =" + count);
+ }
+ makeCMCall(jsonBody, url, debug);
+
+ }
+
+ return totalnbrofrecords;
+
+ }
+
+ public static int protectFromFile(String url, boolean debug, String filePath, String protection_profile)
+ throws Exception {
+
+ JsonObject crdp_payload = new JsonObject();
+ String jsonBody = null;
+ String inputdataarray = null;
+
+ int totalnbrofrecords = 0;
+ try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
+ String line;
+
+ crdp_payload.addProperty("protection_policy_name", protection_profile);
+
+ while ((line = br.readLine()) != null) {
+ totalnbrofrecords++;
+ crdp_payload.addProperty("data", line);
+ inputdataarray = crdp_payload.toString();
+ jsonBody = inputdataarray.replace("{", " ");
+ jsonBody = "{" + jsonBody;
+ if (debug)
+ System.out.println("jsonBody " + jsonBody);
+ makeCMCall(jsonBody, url, debug);
+ crdp_payload = new JsonObject();
+ crdp_payload.addProperty("protection_policy_name", protection_profile);
+ }
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ return totalnbrofrecords;
+
+ }
+
+ public static void disableCertValidation() throws Exception {
+ TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() {
+ public java.security.cert.X509Certificate[] getAcceptedIssuers() {
+ return null;
+ }
+
+ public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) {
+ }
+
+ public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) {
+ }
+ } };
+
+ // Install the all-trusting trust manager
+ try {
+ SSLContext sc = SSLContext.getInstance("SSL");
+ sc.init(null, trustAllCerts, new java.security.SecureRandom());
+ HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ // Create all-trusting host name verifier
+ HostnameVerifier allHostsValid = new HostnameVerifier() {
+ public boolean verify(String hostname, SSLSession session) {
+ return true;
+ }
+ };
+
+ // Install the all-trusting host verifier
+ HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid);
+ }
+
+ public static int makeCMCall(String payload, String url, boolean debug) throws Exception {
+
+ disableCertValidation();
+
+ // Create URL object
+ URL apiUrl = new URL(url);
+ // Create connection object
+ HttpURLConnection connection = (HttpURLConnection) apiUrl.openConnection();
+ connection.setRequestMethod("POST");
+ connection.setRequestProperty("Content-Type", "application/json");
+ connection.setDoOutput(true);
+ // connection.setRequestProperty("Authorization", newtoken);
+ // Send request
+ OutputStream outputStream = connection.getOutputStream();
+ outputStream.write(payload.getBytes());
+ outputStream.flush();
+ outputStream.close();
+
+ // Get response
+ int responseCode = connection.getResponseCode();
+ BufferedReader reader;
+ if (responseCode >= 200 && responseCode < 300) {
+ reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
+ } else {
+ reader = new BufferedReader(new InputStreamReader(connection.getErrorStream()));
+ }
+
+ // Read response
+ String lineresponse;
+ StringBuilder response = new StringBuilder();
+ while ((lineresponse = reader.readLine()) != null) {
+ response.append(lineresponse);
+ }
+ reader.close();
+
+ // Print response
+ if (debug) {
+ System.out.println("Response Code: " + responseCode);
+ System.out.println("Response Body: " + response.toString());
+ }
+ // Disconnect connection
+ connection.disconnect();
+
+ return responseCode;
+
+ }
+
+ public static List readStringsFromFile(String filePath) {
+ List strings = new ArrayList<>();
+
+ try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ strings.add(line);
+ }
+ } catch (IOException e) {
+ System.err.println("Error reading file: " + e.getMessage());
+ }
+
+ return strings;
+ }
+
+}
diff --git a/rest/crdp/python/CRDPBulkOperation.py b/rest/crdp/python/CRDPBulkOperation.py
new file mode 100644
index 00000000..f978c48a
--- /dev/null
+++ b/rest/crdp/python/CRDPBulkOperation.py
@@ -0,0 +1,134 @@
+import json
+import ssl
+import sys
+import requests
+from urllib3 import disable_warnings, exceptions
+from requests.auth import HTTPBasicAuth
+
+BATCH_LIMIT = 5000
+
+def main(args):
+ crdp_container_ip = "yourip"
+ batch_size = 250
+ protect_url = f"http://{crdp_container_ip}:8090/v1/protect"
+ debug = False
+ protection_profile = "plain-nbr-internal"
+ mode_of_operation = "regular"
+
+ if args[1] != "null":
+ debug = args[1].lower() == "true"
+
+ if args[2] != "null":
+ mode_of_operation = args[2]
+
+ if mode_of_operation.lower() == "bulk":
+ protect_url = protect_url + "bulk"
+
+ file_path = r"C:\data\emp-id1k.txt"
+ total_records = 0
+
+ if mode_of_operation.lower() == "regular":
+ total_records = protect_from_file(protect_url, debug, file_path, protection_profile)
+ nbrofchunks = 1
+ else:
+ strings = read_strings_from_file(file_path)
+ total_number_of_records = len(strings)
+ index = 0
+ nbrofchunks = 0
+ records_left = total_number_of_records
+
+ batch_nnn = []
+ if batch_size > total_number_of_records:
+ batch_size = total_number_of_records
+ if batch_size >= BATCH_LIMIT:
+ batch_size = BATCH_LIMIT
+
+ while index < total_number_of_records:
+ for i in range(batch_size):
+ if index < total_number_of_records:
+ batch_nnn.append(strings[index])
+ index += 1
+ records_left -= 1
+
+ total_records += protect_from_file_bulk(protect_url, debug, batch_nnn, batch_size, index, total_number_of_records, records_left, protection_profile)
+ batch_nnn = []
+ nbrofchunks += 1
+
+ print(f"Total records = {total_records}")
+
+def protect_from_file_bulk(url, debug, strings, batch_size, index, total_number_of_records, records_left, protection_profile):
+ total_nb_of_records = 0
+ count = 0
+ crdp_payload_array = []
+
+ for str_ in strings:
+ total_nb_of_records += 1
+ crdp_payload_array.append(str_)
+ count += 1
+
+ if count == batch_size:
+ crdp_payload = {"data_array": crdp_payload_array, "protection_policy_name": protection_profile}
+ json_body = json.dumps(crdp_payload)
+ if debug:
+ print("json body:", json_body)
+ make_cm_call(json_body, url, debug)
+ crdp_payload_array = []
+ count = 0
+
+ if count > 0:
+ crdp_payload = {"data_array": crdp_payload_array, "protection_policy_name": protection_profile}
+ json_body = json.dumps(crdp_payload)
+ if debug:
+ print("json body:", json_body)
+ make_cm_call(json_body, url, debug)
+
+ return total_nb_of_records
+
+def protect_from_file(url, debug, file_path, protection_profile):
+ total_nb_of_records = 0
+
+ with open(file_path, 'r') as file:
+ for line in file:
+ total_nb_of_records += 1
+ crdp_payload = {
+ "protection_policy_name": protection_profile,
+ "data": line.strip()
+ }
+ json_body = json.dumps(crdp_payload)
+ if debug:
+ print("jsonBody", json_body)
+ make_cm_call(json_body, url, debug)
+
+ return total_nb_of_records
+
+def disable_cert_validation():
+ disable_warnings(exceptions.InsecureRequestWarning)
+ ssl._create_default_https_context = ssl._create_unverified_context
+
+def make_cm_call(payload, url, debug):
+ disable_cert_validation()
+
+ headers = {'Content-Type': 'application/json'}
+
+ response = requests.post(url, data=payload, headers=headers, verify=False)
+
+ if debug:
+ print(f"Response Code: {response.status_code}")
+ print(f"Response Body: {response.text}")
+
+ return response.status_code
+
+def read_strings_from_file(file_path):
+ with open(file_path, 'r') as file:
+ return [line.strip() for line in file]
+
+if __name__ == "__main__":
+ if len(sys.argv) < 3:
+ print( len(sys.argv))
+ print("Usage: script.py ")
+ sys.exit(1)
+
+ # sys.argv[0] is the script name, so pass args starting from index 1
+ args = sys.argv[1:]
+ print(f"Debug mode: {args[1]}, Mode of operation: {args[2]}")
+ main(args)
\ No newline at end of file