Update framework to ignore most formatting differences between

framework/src/main/java/org/apache/drill/test/framework/ColumnList.java

@@ -19,6 +19,7 @@
 
 import java.math.BigDecimal;
 import java.sql.Types;
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -91,6 +92,32 @@ public int hashCode() {
     return hash;
   }
 
+  // remove spaces unless they are in a string
+  private List<String> splitStringBySpace(String input) {
+    List<String> result = new ArrayList<String>();
+    int start = 0;
+    boolean inQuotes = false;
+    for (int current = 0; current < input.length(); current++) {
+        if (input.charAt(current) == '\"') inQuotes = !inQuotes; // toggle state
+        boolean atLastChar = (current == input.length() - 1);
+        if(atLastChar) result.add(input.substring(start));
+        else if (input.charAt(current) == ' ' && !inQuotes) {
+            result.add(input.substring(start, current));
+            start = current + 1;
+        }
+    }
+    return result;
+  }
+
+  private String removeStringSpace(String input) {
+    List<String> tokens = splitStringBySpace(input);
+    StringBuilder sb = new StringBuilder();
+    for(String t : tokens) {
+      sb.append(t);
+    }
+    return sb.toString();
+  }
+
   /**
    * String representation of the ColumnList object
    */
@@ -101,14 +128,17 @@ public String toString() {
       int type = (Integer) (types.get(i));
       if (Simba && (type == Types.VARCHAR)) {
         String s1 = String.valueOf(values.get(i));
-        // if the field has a JSON string or list, then remove newlines so that
-        // each record fits on a single line in the actual output file.
-        // sometimes Drill returns records in different orders, and by
-        // ensuring each reoord is on a single line, they can be matched in any
-        // order
+        // Sometimes Simba returns records that span multiple lines.
+        // If the field has a JSON string or list, then remove newlines
+        // so that each record fits on a single line in the actual output
+        // file.  json/json_kvgenflatten/kvgen/kvgen7.q
+        // Also, sometimes Simba returns varchars with white spaces
+        // in different places.  Remove the white space.
+        // convert/convert32.q
         if ((s1.length() > 0) &&
            ((s1.charAt(0) == '{') || (s1.charAt(0) == '[')) ) {
           s1 = Utils.removeNewLines(s1);
+          s1 = removeStringSpace(s1);
           values.set(i, s1);
         }
       }
@@ -117,14 +147,17 @@ public String toString() {
     int type = (Integer) (types.get(values.size()-1));
     if (Simba && (type == Types.VARCHAR)) {
       String s1 = String.valueOf(values.get(values.size()-1));
-      // if the field has a JSON string or list, then remove newlines so that
-      // each record fits on a single line in the actual output file.
-      // sometimes Drill returns records in different orders, and by
-      // ensuring each reoord is on a single line, they can be matched in any
-      // order
+      // Sometimes Simba returns records that span multiple lines.
+      // If the field has a JSON string or list, then remove newlines
+      // so that each record fits on a single line in the actual output
+      // file.  json/json_kvgenflatten/kvgen/kvgen7.q
+      // Also, sometimes Simba returns varchars with white spaces
+      // in different places.  Remove the white space.
+      // convert/convert32.q
       if ((s1.length() > 0) &&
          ((s1.charAt(0) == '{') || (s1.charAt(0) == '[')) ) {
         s1 = Utils.removeNewLines(s1);
+        s1 = removeStringSpace(s1);
         values.set(values.size()-1, s1);
       }
     }

framework/src/main/java/org/apache/drill/test/framework/DrillTestJdbc.java

@@ -153,7 +153,7 @@ public void run() {
 		    }
       }
       if (testStatus == TestStatus.PASS && !TestDriver.cmdParam.outputQueryResult) {
-    	  // Utils.deleteFile(outputFilename);
+    	  Utils.deleteFile(outputFilename);
       }
       duration = stopwatch;
 

framework/src/main/java/org/apache/drill/test/framework/Utils.java

@@ -384,8 +384,13 @@ private static String getExpectedFile(String queryFile, String queryFileExt,
             } else {
               // if expected results file with driver extension exists, then use it
               //     i.e. query.e_tsv.sjdbc
-              // else if expected result file with driver extension and .fail exists, and skip this test
+              // else if expected result file with driver extension and .fail exists,
+              // and skip this test
               //     i.e. query.e_tsv.sjdbc.fail
+              // else if expected result file with driver extension and .fail exists,
+              // and skip this test
+              //     i.e. query.e_tsv.xxx.sjdbc.fail where 'xxx' can be anything, such as a jira
+              //     i.e. query.e_tsv.md4862.sjdbc.fail
               // else use Apache.
               Filename = queryFile.substring(0, idx).concat(expectedFileExt.substring(2));
               FilenameJDBC = Filename.concat(".").concat(driverExt);
@@ -398,7 +403,24 @@ private static String getExpectedFile(String queryFile, String queryFileExt,
                 if (driverFile.exists()) {
                   return null;
                 } else {
-                  return Filename;
+                  File parent = driverFile.getParentFile();
+                  String basename = Filename.substring(Filename.lastIndexOf('/') + 1);
+                  File[] listOfFiles = parent.listFiles();
+                  boolean FOUNDFAILED = false;
+                  for (File file: listOfFiles) {
+                    if (file.isFile()) {
+                      if (file.getName().startsWith(basename) &&
+                          file.getName().endsWith(".sjdbc.fail")) {
+                        FOUNDFAILED = true;
+                        break;
+                      }
+                    }
+                  }
+                  if (FOUNDFAILED) {
+                    return null;
+                  } else {
+                    return Filename;
+                  }
                 }
               }
             }