The awscli tests for virtual style paths (dnsmasq fix) (#278)

- Add new mode to test virtual host style with awscli

Mint awscli tests now run against both path style and virtual style
if virtual style flag is set. By default only path style tests are
run

- The dnsmasq will now resolvethe wildcard entries also

This will now facilitate the randomized wildcard bucket names
in awscli tests (virtual style paths).

Fixes #213

Dockerfile

@@ -10,7 +10,7 @@ ENV GOPATH /usr/local
 
 ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
 
-RUN apt-get --yes update && apt-get --yes upgrade && apt-get --yes --quiet install wget jq curl git && \
+RUN apt-get --yes update && apt-get --yes upgrade && apt-get --yes --quiet install wget jq curl git dnsmasq && \
     git clone https://github.com/minio/mint.git /mint && \
     cd /mint && /mint/release.sh
 

Dockerfile.dev

@@ -12,7 +12,7 @@ ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
 
 WORKDIR /mint
 
-RUN apt-get --yes update && apt-get --yes upgrade && apt-get --yes --quiet install wget jq curl
+RUN apt-get --yes update && apt-get --yes upgrade && apt-get --yes --quiet install wget jq curl dnsmasq
 
 ENV MINT_ROOT_DIR /mint
 ENV MINT_RUN_CORE_DIR $MINT_ROOT_DIR/run/core

README.md

@@ -38,12 +38,28 @@ Below environment variables are required to be passed to the docker container. S
 
 | Environment variable | Description | Example |
 |:--- |:--- |:--- |
-| `SERVER_ENDPOINT` | Endpoint of Minio server in the format `HOST:PORT` | `play.minio.io:9000` |
+| `SERVER_ENDPOINT` | Endpoint of Minio server in the format `HOST:PORT`; for virtual style `IP:PORT` | `play.minio.io:9000` |
 | `ACCESS_KEY` | Access key of access `SERVER_ENDPOINT` | `Q3AM3UQ867SPQQA43P2F` |
 | `SECRET_KEY` | Secret Key of access `SERVER_ENDPOINT` | `zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG` |
 | `ENABLE_HTTPS` | (Optional) Set `1` to indicate to use HTTPS to access `SERVER_ENDPOINT`. Defaults to `0` (HTTP) | `1` |
 | `MINT_MODE` | (Optional) Set mode indicating what category of tests to be run by values `core`, `full` or `worm`.  Defaults to `core` | `full` |
+| `DOMAIN` | (Optional) Value of MINIO_DOMAIN environment variable used in Minio server | `myminio.com` |
+| `ENABLE_VIRTUAL_STYLE` | (Optional) Set `1` to indicate virtual style access . Defaults to `0` (Path style) | `1` |
 
+
+### Test virtual style access against Minio server
+
+To test Minio server virtual style access with Mint, follow these steps:
+
+- Set a domain in your Minio server using environment variable MINIO_DOMAIN. For example `export MINIO_DOMAIN=myminio.com`.
+- Start Minio server. 
+- Execute Mint against Minio server (with `MINIO_DOMAIN` set to `myminio.com`) using this command
+```sh
+$ docker run -e "SERVER_ENDPOINT=192.168.86.133:9000" -e "DOMAIN=minio.com"  \
+	     -e "ACCESS_KEY=minio" -e "SECRET_KEY=minio123" -e "ENABLE_HTTPS=0" \
+	     -e "ENABLE_VIRTUAL_STYLE=1" minio/mint		
+```
+
 ### Mint log format
 
 All test logs are stored in `/mint/log/log.json` as multiple JSON document.  Below is the JSON format for every entry in the log file.

mint.sh

@@ -20,6 +20,7 @@ MINT_DATA_DIR=${MINT_DATA_DIR:-/mint/data}
 MINT_MODE=${MINT_MODE:-core}
 SERVER_REGION=${SERVER_REGION:-us-east-1}
 ENABLE_HTTPS=${ENABLE_HTTPS:-0}
+ENABLE_VIRTUAL_STYLE=${ENABLE_VIRTUAL_STYLE:-0}
 
 if [ -z "$SERVER_ENDPOINT" ]; then
     SERVER_ENDPOINT="play.minio.io:9000"
@@ -28,6 +29,23 @@ if [ -z "$SERVER_ENDPOINT" ]; then
     ENABLE_HTTPS=1
 fi
 
+if [ "$ENABLE_VIRTUAL_STYLE" -eq 1 ]; then
+        SERVER_IP="${SERVER_ENDPOINT%%:*}"
+        SERVER_PORT="${SERVER_ENDPOINT/*:/}"
+        # Check if SERVER_IP is actually IPv4 address
+        octets=("${SERVER_IP//./ }")
+        if [ "${#octets[@]}" -ne 4 ]; then
+            echo "$SERVER_IP must be an IP address"
+            exit 1
+        fi
+        for octet in "${octets[@]}"; do
+	    if [ "$octet" -lt 0 ] 2>/dev/null || [ "$octet" -gt 255 ] 2>/dev/null; then
+		echo "$SERVER_IP must be an IP address"
+		exit 1
+	    fi
+        done
+fi
+
 ROOT_DIR="$PWD"
 TESTS_DIR="$ROOT_DIR/run/core"
 
@@ -84,7 +102,6 @@ function run_test()
             jq . <<<"$entry"
         fi
     fi
-
     return $rv
 }
 
@@ -93,23 +110,27 @@ function main()
     export MINT_DATA_DIR
     export MINT_MODE
     export SERVER_ENDPOINT
+    export SERVER_IP
+    export SERVER_PORT
+
     export ACCESS_KEY
     export SECRET_KEY
     export ENABLE_HTTPS
     export SERVER_REGION
-
+    export ENABLE_VIRTUAL_STYLE
+
     echo "Running with"
-    echo "SERVER_ENDPOINT: $SERVER_ENDPOINT"
-    echo "ACCESS_KEY:      $ACCESS_KEY"
-    echo "SECRET_KEY:      ***REDACTED***"
-    echo "ENABLE_HTTPS:    $ENABLE_HTTPS"
-    echo "SERVER_REGION:   $SERVER_REGION"
-    echo "MINT_DATA_DIR:   $MINT_DATA_DIR"
-    echo "MINT_MODE:       $MINT_MODE"
+    echo "SERVER_ENDPOINT:      $SERVER_ENDPOINT"
+    echo "ACCESS_KEY:           $ACCESS_KEY"
+    echo "SECRET_KEY:           ***REDACTED***"
+    echo "ENABLE_HTTPS:         $ENABLE_HTTPS"
+    echo "SERVER_REGION:        $SERVER_REGION"
+    echo "MINT_DATA_DIR:        $MINT_DATA_DIR"
+    echo "MINT_MODE:            $MINT_MODE"
+    echo "ENABLE_VIRTUAL_STYLE: $ENABLE_VIRTUAL_STYLE"
     echo
-
     echo "To get logs, run 'docker cp ${CONTAINER_ID}:/mint/log /tmp/mint-logs'"
-
+    
     declare -a run_list
     if [ "$MINT_MODE" == "worm" ]; then
         if [ "$#" -gt 1 ]; then
@@ -135,7 +156,7 @@ function main()
             run_list=( "${run_list[@]}" "$TESTS_DIR/$sdk" )
         done
     fi
-
+    
     count="${#run_list[@]}"
     i=0
     for sdk_dir in "${run_list[@]}"; do
@@ -151,7 +172,7 @@ function main()
             break
         fi
     done
-
+    
     ## Report when all tests in run_list are run
     if [ $i -eq "$count" ]; then
         echo -e "\nAll tests ran successfully"
@@ -160,5 +181,4 @@ function main()
         exit 1
     fi
 }
-
 main "$@"

run/core/awscli/run.sh

@@ -29,9 +29,23 @@ aws configure set aws_access_key_id "$ACCESS_KEY"
 aws configure set aws_secret_access_key "$SECRET_KEY"
 aws configure set default.region "$SERVER_REGION"
 
-# run tests
+# run tests for virtual style if provided
+if [ "$ENABLE_VIRTUAL_STYLE" -eq 1 ]; then
+   # Setup endpoint scheme
+   endpoint="http://$DOMAIN:$SERVER_PORT"
+   if [ "$ENABLE_HTTPS" -eq 1 ]; then
+       endpoint="https://$DOMAIN:$SERVER_PORT"
+   fi
+   dnsmasq --address="/$DOMAIN/$SERVER_IP" --user=root
+   echo -e "nameserver 127.0.0.1\n$(cat /etc/resolv.conf)" > /etc/resolv.conf
+   aws configure set default.s3.addressing_style virtual
+   ./test.sh "$endpoint"  1>>"$output_log_file" 2>"$error_log_file"
+   aws configure set default.s3.addressing_style path
+fi
+
 endpoint="http://$SERVER_ENDPOINT"
 if [ "$ENABLE_HTTPS" -eq 1 ]; then
     endpoint="https://$SERVER_ENDPOINT"
 fi
+# run path style tests
 ./test.sh "$endpoint"  1>>"$output_log_file" 2>"$error_log_file"