About a year and a half ago, we delivered this automated installer w…

…hich can install open-source ranger, as emr-native ranger feature release, we deeply refactored it, combined open-source and emr-native ranger together, make it support 4 high applicability scenarios ( No.1, 2, 3, 4) altogether. for this commit, Scenario 1: "OpenLDAP + EMR-Native Ranger" is fully tested, works very well.

bin/ad.ps1

@@ -0,0 +1,61 @@
+# 安装Active Directory服务
+Install-windowsfeature -name AD-Domain-Services -IncludeManagementTools
+
+# 创建Forest, 命令执行成功之后会自动重启服务器，虽然使用-NoRebootOnCompletion:$true可以规避自动重启，但是如果要正常使用AD服务，还是要重启服务器。
+Install-ADDSForest -DomainName example.com -SafeModeAdministratorPassword (ConvertTo-SecureString 'Admin1234!' -AsPlainText -Force) -DomainMode WinThreshold -DomainNetbiosName ABC -ForestMode WinThreshold -DatabasePath "C:\Windows\NTDS" -LogPath "C:\Windows\NTDS" -SysvolPath "C:\Windows\SYSVOL" -CreateDnsDelegation:$false -InstallDns:$true -NoRebootOnCompletion:$false -Force:$true
+
+ksetup /addkdc CN-NORTH-1.COMPUTE.INTERNAL
+
+netdom trust CN-NORTH-1.COMPUTE.INTERNAL /Domain:EXAMPLE.COM /add /realm /passwordt:Admin1234!
+
+# 执行该命令前，需在AD服务上手都配置安全组策略, 确保“Kerberos允许的加密类型”这一配置项包含如下两种加密类型中的一种或全部！
+ksetup /SetEncTypeAttr CN-NORTH-1.COMPUTE.INTERNAL AES256-CTS-HMAC-SHA1-96 AES128-CTS-HMAC-SHA1-96
+
+# ------------------------------------------   ou: Service Accounts   ------------------------------------------- #
+
+# Remove OU "Service Accounts" recursively if exists
+# Get-ADOrganizationalUnit -Identity "OU=Service Accounts,DC=EXAMPLE,DC=COM" |
+# Set-ADObject -ProtectedFromAccidentalDeletion:$false -PassThru |
+# Remove-ADOrganizationalUnit -Confirm:$false -Recursive
+
+# Add OU "Service Accounts"
+New-ADOrganizationalUnit -Name "Service Accounts" -Path "DC=EXAMPLE,DC=COM"
+
+# -------------------------------------------   user: ranger-binder   ------------------------------------------- #
+
+# Remove service account "ranger-binder" if exists
+Remove-ADUser -Identity "CN=ranger-binder,OU=Service Accounts,DC=EXAMPLE,DC=COM" -Confirm:$false
+
+# Create service account "ranger-binder"
+New-ADUser -Name "ranger-binder" -OtherAttributes @{'title'="ranger-binder"} -path "OU=Service Accounts,DC=EXAMPLE,DC=COM"
+Set-ADAccountPassword -Identity "CN=ranger-binder,OU=Service Accounts,DC=EXAMPLE,DC=COM" -Reset -NewPassword (ConvertTo-SecureString -AsPlainText 'Admin1234!' -Force)
+Set-ADUser -Identity "CN=ranger-binder,OU=Service Accounts,DC=EXAMPLE,DC=COM" -PasswordNeverExpires $true
+Enable-ADAccount -Identity "CN=ranger-binder,OU=Service Accounts,DC=EXAMPLE,DC=COM"
+
+# ----------------------------------------   user: emr-ad-domain-joiner   --------------------------------------- #
+
+# Remove service account "emr-ad-domain-joiner" if exists
+Remove-ADUser -Identity "CN=emr-ad-domain-joiner,OU=Service Accounts,DC=EXAMPLE,DC=COM" -Confirm:$false
+
+# Create service account "emr-ad-domain-joiner"
+New-ADUser -Name "emr-ad-domain-joiner" -OtherAttributes @{'title'="emr-ad-domain-joiner"} -path "OU=Service Accounts,DC=EXAMPLE,DC=COM"
+Set-ADAccountPassword -Identity "CN=emr-ad-domain-joiner,OU=Service Accounts,DC=EXAMPLE,DC=COM" -Reset -NewPassword (ConvertTo-SecureString -AsPlainText 'Admin1234!' -Force)
+Set-ADUser -Identity "CN=emr-ad-domain-joiner,OU=Service Accounts,DC=EXAMPLE,DC=COM" -PasswordNeverExpires $true
+Enable-ADAccount -Identity "CN=emr-ad-domain-joiner,OU=Service Accounts,DC=EXAMPLE,DC=COM"
+
+# Add service account "emr-ad-domain-joiner" to "Domain Admins", "Schema Admins", "Enterprise Admins" groups,
+# so as it has enough privileges to add EMR cluster nodes into AD domain.
+Add-ADGroupMember -Identity "Domain Admins" -Members "CN=emr-ad-domain-joiner,OU=Service Accounts,DC=EXAMPLE,DC=COM"
+Add-ADGroupMember -Identity "Schema Admins" -Members "CN=emr-ad-domain-joiner,OU=Service Accounts,DC=EXAMPLE,DC=COM"
+Add-ADGroupMember -Identity "Enterprise Admins" -Members "CN=emr-ad-domain-joiner,OU=Service Accounts,DC=EXAMPLE,DC=COM"
+
+# ---------------------------------------------   user: ad-user-1   --------------------------------------------- #
+
+# Remove service account "emr-ad-domain-joiner" if exists
+Remove-ADUser -Identity "CN=ad-user-1,CN=Users,DC=EXAMPLE,DC=COM" -Confirm:$false
+
+# Create normal domain user  "ad-user-1"
+New-ADUser -Name "ad-user-1" -OtherAttributes @{'title'="ad-user-1"} -path "CN=Users,DC=EXAMPLE,DC=COM"
+Set-ADAccountPassword -Identity "CN=ad-user-1,CN=Users,DC=EXAMPLE,DC=COM" -Reset -NewPassword (ConvertTo-SecureString -AsPlainText 'Admin1234!' -Force)
+Set-ADUser -Identity "CN=ad-user-1,CN=Users,DC=EXAMPLE,DC=COM" -PasswordNeverExpires $true
+Enable-ADAccount -Identity "CN=ad-user-1,CN=Users,DC=EXAMPLE,DC=COM"

bin/crt.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+
+#
+# This script apply follow naming role:
+# for a principal, i.e. ranger-admin, its:
+# 1. private key: ranger-admin.key
+# 2. certificate: ranger-admin.crt
+# 3. pcks12 file: ranger-admin.p12
+# 4. keystore:    ranger-admin-keystore.jks
+# 5. truststore:  ranger-admin-truststore.jks
+#
+
+createRangerSecrets() {
+    printHeading "CREATE KEY, CRT, KEYSTORE & TRUSTSTORE FOR RANGER"
+
+    rm -rf $RANGER_SECRETS_DIR
+    mkdir -p $RANGER_SECRETS_DIR
+
+    # ranger-admin
+    printHeading "CREATE KEY, CRT, KEYSTORE FOR [ ranger-admin ]"
+    createKeyCertPair "ranger-admin"
+    createKeystore "ranger-admin"
+
+    # ranger-plugin
+    #   it is NOT possible to use ranger-plugin certs, because they will be generated & managed by EMR
+    #   but for other open-source plugins, i.e. HBase, Presto, these certs will be required if ranger admin https enabled!
+    printHeading "CREATE KEY, CRT, KEYSTORE FOR [ ranger-plugin ]"
+    createKeyCertPair "ranger-plugin"
+    createKeystore "ranger-plugin"
+
+    # build truststore file used by ranger admin, be careful, this file NOT USED!
+    # instead, plugins crts will install to JVM cacerts file.
+    createOrImportToTruststore "ranger-admin" "ranger-admin"# trust self
+    createOrImportToTruststore "ranger-admin" "ranger-plugin" # trust plugin
+
+    # build truststore file used by ranger open-source plugins.
+    # for emr-native plugins, EMR will regenerate truststore with key & certs from secrets manager.
+    createOrImportToTruststore "ranger-plugin" "ranger-plugin" # trust self
+    createOrImportToTruststore "ranger-plugin" "ranger-admin" # trust admin
+
+    # NOTE: because there is explicit truststore config items in ranger admin,
+    # there is no way to set the path of 'ranger-admin-truststore.jks',
+    # so, we HAVE TO add all trusted crts to JAVA default crts store: $JAVA_HOME/jre/lib/security/cacerts
+    removeFromJavaDefaultTruststore "ranger-admin"
+    importToJavaDefaultTruststore "ranger-admin" # trust self
+    removeFromJavaDefaultTruststore "ranger-plugin"
+    importToJavaDefaultTruststore "ranger-plugin" # trust self
+
+    listCerts "ranger-admin"
+    listCerts "ranger-plugin"
+
+    removeSecretsOnSecretsManager
+    uploadRangerSecretsToSecretsManager
+}
+
+removeRangerSecrets() {
+    removeSecretsOnSecretsManager
+    removeFromJavaDefaultTruststore "ranger-admin"
+    removeFromJavaDefaultTruststore "ranger-plugin"
+    rm -rf $RANGER_SECRETS_DIR
+}
+
+createKeyCertPair() {
+    principal="$1"
+    openssl req -x509 -newkey rsa:2048 -nodes -keyout $RANGER_SECRETS_DIR/${principal}.key -out $RANGER_SECRETS_DIR/${principal}.crt -days 3650  -subj "/C=CN/ST=SHANGHAI/L=SHANGHAI/O=EXAMPLE/OU=IT/CN=$CERTIFICATE_CN"
+}
+
+createKeystore() {
+    principal="$1"
+    openssl pkcs12 -export -in $RANGER_SECRETS_DIR/${principal}.crt -inkey $RANGER_SECRETS_DIR/${principal}.key -chain -CAfile $RANGER_SECRETS_DIR/${principal}.crt -name ${principal} -out $RANGER_SECRETS_DIR/${principal}.p12 -password pass:changeit
+    keytool -importkeystore -deststorepass changeit -destkeystore $RANGER_SECRETS_DIR/${principal}-keystore.jks -srckeystore $RANGER_SECRETS_DIR/${principal}.p12 -srcstoretype PKCS12 -srcstorepass changeit
+}
+
+createOrImportToTruststore() {
+    trustingPrincipal="$1"
+    trustedPrincipal="$2"
+    keytool -import -file $RANGER_SECRETS_DIR/${trustedPrincipal}.crt -alias $trustedPrincipal -keystore $RANGER_SECRETS_DIR/${trustingPrincipal}-truststore.jks -storepass changeit -noprompt
+}
+
+removeFromJavaDefaultTruststore() {
+    trustedPrincipal="$1"
+    keytool -delete -alias $trustedPrincipal -keystore $JAVA_HOME/jre/lib/security/cacerts -storepass changeit
+}
+
+importToJavaDefaultTruststore() {
+    trustedPrincipal="$1"
+    keytool -import -file $RANGER_SECRETS_DIR/${trustedPrincipal}.crt -alias $trustedPrincipal -keystore $JAVA_HOME/jre/lib/security/cacerts -storepass changeit -noprompt
+}
+
+listCerts() {
+    principal="$1"
+    echo "${principal}-keystore.jks contains following certificates:"
+    keytool -list -v -keystore $RANGER_SECRETS_DIR/${principal}-keystore.jks -storepass changeit|grep Alias.*
+    echo "${principal}-truststore.jks contains following certificates:"
+    keytool -list -v -keystore $RANGER_SECRETS_DIR/${principal}-truststore.jks -storepass changeit|grep Alias.*
+}
+
+uploadRangerSecretsToSecretsManager() {
+    printHeading "UPLOAD SECRETS TO SECRETSMANAGER"
+    # append postfix for secret id in case there are multiple clusters to be created!
+    aws secretsmanager create-secret --name "ranger-admin@${RANGER_HOST}" --description "Ranger Admin Certificate" --secret-string file://$RANGER_SECRETS_DIR/ranger-admin.crt --region $REGION
+    # EMR Security Configuration need a secret file contains plugin key and cert both.
+    cat $RANGER_SECRETS_DIR/ranger-plugin.key $RANGER_SECRETS_DIR/ranger-plugin.crt >  $RANGER_SECRETS_DIR/ranger-plugin.key+crt
+    aws secretsmanager create-secret --name "ranger-plugin@${RANGER_HOST}" --description "Ranger Admin Private Key & Certificate" --secret-string file://$RANGER_SECRETS_DIR/ranger-plugin.key+crt --region $REGION
+}
+
+removeSecretsOnSecretsManager() {
+    printHeading "REMOVE SECRETS FROM SECRETSMANAGER"
+    aws secretsmanager delete-secret --secret-id "ranger-admin@${RANGER_HOST}" --force-delete-without-recovery --region $REGION --cli-read-timeout 10 --cli-connect-timeout 10
+    aws secretsmanager delete-secret --secret-id "ranger-plugin@${RANGER_HOST}" --force-delete-without-recovery --region $REGION --cli-read-timeout 10 --cli-connect-timeout 10
+
+    # deleting secrets is async job, have to wait for a while...
+    sleep 30
+}
+

bin/ec2.sh

@@ -0,0 +1,127 @@
+#!/bin/bash
+
+initEc2() {
+    # init ec2 is a one-time job, no need to run duplicatly,
+    # especially when re-run all-in-one install
+    if [ -f $INIT_EC2_FLAG_FILE ]; then
+        echo "This ec2 instance has been initialized, nothing to do!"
+        echo "If you want to force re-init this ec2, please execute force-init-ec2 command"
+    else
+        if [[ "$REGION" = "" || "$ACCESS_KEY_ID" = "" || "$SECRET_ACCESS_KEY" = "" ]]; then
+            echo "ERROR! --region or --access-key-id or --secret-access-key is not provided!"
+            exit 1
+        fi
+        installTools
+        configSsh
+        installAwsCli
+        installJdk8IfNotExists
+        touch $INIT_EC2_FLAG_FILE
+    fi
+}
+
+forceInitEc2() {
+    rm -f $INIT_EC2_FLAG_FILE
+    initEc2
+}
+
+installTools() {
+    printHeading "INSTALL COMMON TOOLS ON EC2"
+    yum -y update
+    # install common tools
+    yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-$(rpm -E '%{rhel}').noarch.rpm
+    yum -y install lrzsz vim wget zip unzip expect tree htop iotop nc telnet jq
+
+    # change timezone
+    cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+}
+
+configSsh() {
+    printHeading "CONFIG SSH"
+    # enable ssh login with password
+    sed -i "s/PasswordAuthentication no/PasswordAuthentication yes/g" /etc/ssh/sshd_config
+    echo "PermitRootLogin yes" | tee -a  /etc/ssh/sshd_config
+    echo "RSAAuthentication yes" | tee -a  /etc/ssh/sshd_config
+    systemctl restart sshd
+
+    # overwrite authorized_keys of root, because it blocks root login with: "no-port-forwarding, ... echo;sleep 10""
+    # if user: hadoop exists, this is a node of EMR, otherwise, this is a normal EC2 instance!
+    egrep "^hadoop\:" /etc/passwd >& /dev/null
+    if [ "$?" == "0" ]; then
+        cat /home/hadoop/.ssh/authorized_keys > /root/.ssh/authorized_keys
+    else
+        cat /home/ec2-user/.ssh/authorized_keys > /root/.ssh/authorized_keys
+    fi
+}
+
+installAwsCli() {
+    printHeading "INSTALL AWS CLI V2"
+    # aws cli is very stupid!
+    # for v1, it is installed via rpm/yum, so with 'yum list installed awscli', we can't get version
+    # for v2, it is installed via zip Packages, it does NOT work with 'yum list installed awscli', only 'aws --version' works
+    # but for v1, 'aws --version' does not work, it DO print version message, but does NOT return string value!
+    # if let message=$(aws --version), it prints message on console, but $message is empty!
+    # so, it is REQUIRED to append '2>&1', the following is right way to get version:
+    # awscliVer=$(aws --version 2>&1 | grep -o '[0-9]*\.[0-9]*\.[0-9]*' | head -n1)
+    rm /tmp/awscli -rf
+    echo "Remove awscli v1 if exists ..."
+    yum -y remove awscli
+
+    echo "Remove awscli v2 if exists in case not latest version ..."
+    rm /usr/bin/aws
+    rm /usr/local/bin/aws
+    rm /usr/bin/aws_completer
+    rm /usr/local/bin/aws_completer
+    rm -rf /usr/local/aws-cli
+
+    echo "Install latest awscli v2 ..."
+    wget "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -P "/tmp/awscli/"  &> /dev/null
+    unzip /tmp/awscli/awscli-exe-linux-x86_64.zip -d /tmp/awscli/ &> /dev/null
+    /tmp/awscli/aws/install
+    ln -s /usr/local/bin/aws /usr/bin/aws
+
+    echo "Create credentials for awscli ..."
+    tee /tmp/config <<EOF
+[default]
+region = $REGION
+EOF
+    tee /tmp/credentials <<EOF
+[default]
+aws_access_key_id = $ACCESS_KEY_ID
+aws_secret_access_key = $SECRET_ACCESS_KEY
+EOF
+    # for non-root users
+    users=(ec2-user hadoop)
+    for user in "${users[@]}"; do
+        # if user exists, add awscli credentials
+        egrep "^$user\:" /etc/passwd >& /dev/null
+        if [ "$?" == "0" ]; then
+            rm -rf /home/$user/.aws
+            mkdir /home/$user/.aws
+            cp /tmp/config /home/$user/.aws/
+            cp /tmp/credentials /home/$user/.aws/
+            chown -R $user:$user /home/$user/.aws
+        fi
+    done
+    # for root user
+    rm -rf /root/.aws
+    mkdir /root/.aws
+    cp /tmp/config /root/.aws/
+    cp /tmp/credentials /root/.aws/
+    chown -R root:root /root/.aws
+
+    rm -f /tmp/config /tmp/credentials
+}
+
+installJdk8IfNotExists() {
+    rpm -q java-1.8.0-openjdk-devel &>/dev/null
+    if [ ! "$?" = "0" ]; then
+        printHeading "INSTALL OPEN JDK8"
+        yum -y install java-1.8.0-openjdk-devel
+        printHeading "MAKE AND EXPORT JAVA ENV VARS"
+        echo "export JAVA_HOME=$JAVA_HOME;export PATH=$JAVA_HOME/bin:$PATH" > /etc/profile.d/java.sh
+        source /etc/profile.d/java.sh
+        # make sure ec2-user also set JAVA_HOME, only works after re-login as ec2-user
+        # not work for current ssh session!
+        sudo -i -u ec2-user source /etc/profile.d/java.sh
+    fi
+}