Missing bracket (#35)

* Fix for double brackets

Fixes the issues related to brackets #24

* Run all those constraints

Fixes the issue #31, This prevents from the program to fail when it has issue running constraint related queries.

If there is a error executing a query to fix the constraint, throw the error on the screen and continue till the end.

At the end the user can manually fix anything if there is any issue.

* Bump the version number

README.md

@@ -130,10 +130,11 @@ For more examples how to use the tool, please check out the [wiki](https://githu
 
 # Known Issues
 
-1. If you have a composite unique index where one column is part of foreign key column then there are chances the constraint creation would fail.
-2. Fixing CHECK constraints isn't supported due to complexity, so recreating check constraints would fail, use `custom` subcommand to control the data being inserted
-3. On Greenplum Database partition tables are not supported (due to check constraint issues defined above), so use the `custom` sub command to define the data to be inserted to the column with check constraints
-4. Custom data types are not supported, use `custom` sub command to control the data for that custom data types
+1. We do struggle when recreating constraints, even though we do try to fix the primary key , foreign key, unique key. So there is no guarantee that the tool will fix all the constraints and manual intervention is needed in some cases.
+2. If you have a composite unique index where one column is part of foreign key column then there are chances the constraint creation would fail.
+3. Fixing CHECK constraints isn't supported due to complexity, so recreating check constraints would fail, use `custom` subcommand to control the data being inserted
+4. On Greenplum Database partition tables are not supported (due to check constraint issues defined above), so use the `custom` sub command to define the data to be inserted to the column with check constraints
+5. Custom data types are not supported, use `custom` sub command to control the data for that custom data types
 
 # Developers / Collaboration
 

constraintsRestore.go

@@ -9,6 +9,7 @@ var (
 	ignoreErr = []string{
 		"ERROR #42P16 multiple primary keys for table",
 		"already exists"}
+	maxLoop = 10
 )
 
 // Get Foriegn key objects
@@ -49,32 +50,35 @@ func FixConstraints() {
 func fixPKey(pk constraint) {
 	Debugf("Fixing the Primary / Unique Key for table %s", pk.table)
 	totalViolators := 1
-
+	totalLoop := 0
+
 	// Extract the columns from the list that was collected during backup
-	keys, err := ColExtractor(pk.column, `\(.*?\)`)
+	keys, err := ColExtractor(pk.column, `\(([^\[\]]*)\)`)
 	if err != nil {
 		Fatalf("unable to determine PK violators key columns: %v", err)
 	}
-	cols := strings.Trim(keys, "()")
+	cols := TrimPrefixNSuffix(RemoveEverySuffixAfterADelimiter(keys, " where "), "(", ")")
 
-	for totalViolators > 0 { // Loop till we get a 0 value (i.e 0 violation )
+	for totalViolators > 0 && totalLoop <= maxLoop { // Loop till we get a 0 value (i.e 0 violation ) or max 10 loops
 		// How many violations are we having, if zero then loop breaks
 		totalViolators = getTotalPKViolator(pk.table, cols)
 		if totalViolators > 0 { // Found violation, time to fix it
 
 			// If there are two or more columns forming a PK or UK
-			// lets only fix column by column.
-			totalColumns := strings.Split(cols, ",")
+			pkColumns := strings.Split(cols, ",")
 
 			// Get data type associated with the data types
-			dTypes := getDatatype(pk.table, totalColumns)
+			dTypes := getDatatype(pk.table, pkColumns)
 
 			//Fix the primary constraints by picking the columns from the
 			//array, i.e we update the column one by one.
 			for _, v := range dTypes {
 				fixPKViolator(pk.table, v.Colname, v.Dtype)
 			}
 		}
+		// If there is still violation the function deleteViolatingPkOrUkConstraints takes
+		// care of it
+		totalLoop++
 	}
 }
 
@@ -109,6 +113,15 @@ func fixFKey(con constraint) {
 	Debugf("Checking / Fixing FOREIGN KEY Violation table: %s, column: %s, reference: %s(%s)",
 		fkeyObjects.Table, fkeyObjects.Column, fkeyObjects.Reftable, fkeyObjects.Refcolumn)
 
+	// If its a composite FK relations then pick only one and fix it
+	// TODO: This is a bad logic, this will not quarantee a fix for the composite
+	// TODO: key.  Clean this out later when get a proper solution to overcome
+	// TODO: the composite key
+	col := strings.Split(fkeyObjects.Column, ",")
+	refCol := strings.Split(fkeyObjects.Refcolumn, ",")
+	fkeyObjects.Column = col[0]
+	fkeyObjects.Refcolumn = refCol[0]
+
 	// Loop till we reach the the end of the loop
 	for totalViolators > 0 {
 
@@ -139,7 +152,7 @@ func getForeignKeyColumns(con constraint) *ForeignKey {
 	if err != nil {
 		Fatalf("Unable to extract foreign key column from fk clause: %v", err)
 	}
-	fkCol = strings.Trim(fkCol, "()")
+	fkCol = TrimPrefixNSuffix(fkCol, "(", ")")
 
 	// Extract the reference column from the clause
 	refCol, err := ColExtractor(refClause, `\(.*?\)`)
@@ -150,7 +163,7 @@ func getForeignKeyColumns(con constraint) *ForeignKey {
 	// Extract reference table from the clause
 	refTab := strings.Replace(refClause, refCol, "", -1)
 	refTab = strings.Replace(refTab, "REFERENCES ", "", -1)
-	refCol = strings.Trim(refCol, "()")
+	refCol = TrimPrefixNSuffix(refCol, "(", ")")
 
 	return &ForeignKey{con.table, fkCol, refTab, refCol}
 }
@@ -187,7 +200,7 @@ func recreateAllConstraints() {
 			}
 
 			// Start the progress bar
-			bar := StartProgressBar(fmt.Sprintf("Recreated the Constraint Type \"%s\"", con), len(contents))
+			bar := StartProgressBar(fmt.Sprintf("Recreating the constraint type \"%s\"", con), len(contents))
 
 			// Recreate all constraints one by one, if we can't create it then display the message
 			// on the screen and continue with the rest, since we don't want it to fail if we cannot
@@ -198,7 +211,7 @@ func recreateAllConstraints() {
 					Debugf("Error creating constraint %s, err: %v", content, err)
 					// Try an attempt to recreate constraint again after deleting the
 					// violating row
-					successOrFailure := deleteViolatingPkOrUkConstriants(content)
+					successOrFailure := deleteViolatingPkOrUkConstraints(content)
 					if !successOrFailure { // didn't succeed, ask the user to fix it manually
 						err = WriteToFile(failedConstraintsFile, content+"\n")
 						if err != nil {
@@ -226,15 +239,13 @@ func recreateAllConstraints() {
 // is, we will delete the rows that violates it and hoping that it will help in
 // recreating the constraints. Yes we will loose that row at least that help to
 // recreate constraints ( fingers crossed :) )
-func deleteViolatingPkOrUkConstriants(con string) bool {
+func deleteViolatingPkOrUkConstraints(con string) bool {
 	Debugf("Attempting to run the constraint command %s second time, after deleting violating rows", con)
 	// does the DDL contain PK or UK keyword then do the following
 	// rest send them back for user to fix it.
-	if isSubStringAvailableOnString(con, "ADD CONSTRAINT.*PRIMARY KEY|ADD CONSTRAINT.*UNIQUE") {
-		column, _ := ColExtractor(con, `\(.*?\)`)
-		table, _ := ColExtractor(con, `ALTER TABLE(.*)ADD CONSTRAINT`)
-		table = strings.Trim(strings.Trim(table, "ALTER TABLE"), "ADD CONSTRAINT")
-		column = strings.Trim(column, "()")
+	if isSubStringAvailableOnString(con, "ADD CONSTRAINT.*PRIMARY KEY|ADD CONSTRAINT.*UNIQUE|CREATE UNIQUE INDEX") {
+		// Extract the table and column name
+		table, column := ExtractTableNColumnName(con)
 		err := deleteViolatingConstraintKeys(table, column)
 		if err != nil { // we failed to delete the the constraint violation rows
 			Debugf("Error when deleting rows from the constraint violation rows: %v", err)
@@ -250,3 +261,30 @@ func deleteViolatingPkOrUkConstriants(con string) bool {
 	}
 	return false
 }
+
+// Extract the table name and the column from the sql command
+func ExtractTableNColumnName(s string) (string, string) {
+	var isItAlterStatement bool = true
+	var table string
+	var column string
+
+	// Is this a create statement
+	if strings.HasPrefix(s, "CREATE UNIQUE") { // like create unique index statement
+		isItAlterStatement = false
+	}
+
+	// Extract the column name
+	column, _ = ColExtractor(s, `\(([^\[\]]*)\)`)
+	column = TrimPrefixNSuffix(RemoveEverySuffixAfterADelimiter(column, " where "), "(", ")")
+
+	// Extract the table name
+	switch isItAlterStatement {
+	case true:
+		table, _ = ColExtractor(s, `ALTER TABLE(.*)ADD CONSTRAINT`)
+		table = TrimPrefixNSuffix(table, "ALTER TABLE", "ADD CONSTRAINT")
+	case false:
+		table, _ = ColExtractor(s, `ON(.*)USING`)
+		table = TrimPrefixNSuffix(table, "ON", "USING")
+	}
+	return table, column
+}

helpers.go

@@ -206,6 +206,23 @@ func ColExtractor(conkey, regExp string) (string, error) {
 	return "", nil
 }
 
+// Trim brackets at the start and at the end
+func TrimPrefixNSuffix(s, prefix, suffix string) string {
+	return strings.TrimPrefix(strings.TrimSuffix(s, suffix) , prefix)
+}
+
+// Remove everything after a delimiter
+func RemoveEverySuffixAfterADelimiter(s string, d string) string {
+	// Protect from upper case and lower case bugs
+	s = strings.ToLower(s)
+	d = strings.ToLower(d)
+	if strings.Contains(s, d) {
+		spiltString := strings.Split(s, d)
+		return spiltString[0]
+	}
+	return s
+}
+
 // If given a datatype see if it has a bracket or not.
 func BracketsExists(dt string) bool {
 	var rgx = regexp.MustCompile(`\(.*\)`)
@@ -263,4 +280,11 @@ func CharLen(dt string) (int, error) {
 		returnValue = 1
 	}
 	return returnValue, nil
+}
+
+// New line if its not a debug
+func addNewLine() {
+	if !cmdOptions.Debug {
+		fmt.Println()
+	}
 }

mock.go

@@ -7,7 +7,7 @@ import (
 
 var (
 	programName        = "mock"
-	programVersion     = "v2.5"
+	programVersion     = "v2.6"
 	ExecutionTimestamp = TimeNow()
 	Path               = fmt.Sprintf("%s/%s/%s", os.Getenv("HOME"), programName, ExecutionTimestamp)
 )

sql.go

@@ -412,8 +412,9 @@ func getTotalPKViolator(tab, cols string) int {
 
 	_, err := db.Query(pg.Scan(&total), query)
 	if err != nil {
+		addNewLine()
 		Debugf("query: %s", query)
-		Fatalf("Error when executing the query to extract pk violators: %v", err)
+		Errorf("Error when executing the query to extract pk violators: %v", err)
 	}
 
 	return total
@@ -436,8 +437,9 @@ func GetPKViolators(tab, cols string) []DBViolationRow {
 	query := strings.Replace(getPKViolator(tab, cols), "SELECT "+cols, "SELECT "+cols+" AS row", -1)
 	_, err := db.Query(&result, query)
 	if err != nil {
+		addNewLine()
 		Debugf("query: %s", query)
-		Fatalf("Error when executing the query to extract pk violators for table %s: %v", tab, err)
+		Errorf("Error when executing the query to extract pk violators for table %s: %v", tab, err)
 	}
 
 	return result
@@ -457,9 +459,9 @@ WHERE  ctid =
 	query = fmt.Sprintf(query, tab, col, newdata, whichrow)
 	_, err := ExecuteDB(query)
 	if err != nil {
-		fmt.Println()
+		addNewLine()
 		Debugf("query: %s", query)
-		Fatalf("Error when updating the primary key for table %s, err: %v", tab, err)
+		Errorf("Error when updating the primary key for table %s, err: %v", tab, err)
 	}
 	return ""
 }
@@ -489,8 +491,9 @@ func GetTotalFKViolators(key ForeignKey) int {
 
 	_, err := db.Query(pg.Scan(&total), query)
 	if err != nil {
+		addNewLine()
 		Debugf("Query: %s", query)
-		Fatalf("Error when executing the query to total rows of foreign keys for table %s: %v", key.Table, err)
+		Errorf("Error when executing the query to total rows of foreign keys for table %s: %v", key.Table, err)
 	}
 
 	return total
@@ -507,9 +510,9 @@ func TotalRows(tab string) int {
 
 	_, err := db.Query(pg.Scan(&total), query)
 	if err != nil {
-		fmt.Println()
+		addNewLine()
 		Debugf("query: %s", query)
-		Fatalf("Error when executing the query to total rows: %v", err)
+		Errorf("Error when executing the query to total rows: %v", err)
 	}
 
 	return total
@@ -527,8 +530,9 @@ func GetFKViolators(key ForeignKey) []DBViolationRow {
 	query := strings.Replace(getFKViolators(key), "SELECT "+key.Column, "SELECT "+key.Column+" AS row", -1)
 	_, err := db.Query(&result, query)
 	if err != nil {
+		addNewLine()
 		Debugf("query: %s", query)
-		Fatalf("Error when executing the query to extract fk violators for table %s: %v", key.Table, err)
+		Errorf("Error when executing the query to extract fk violators for table %s: %v", key.Table, err)
 	}
 
 	return result
@@ -547,15 +551,15 @@ WHERE  %[2]s = '%[6]s'
 	query = fmt.Sprintf(query, key.Table, key.Column, key.Refcolumn, key.Reftable, totalRows, whichRow)
 	_, err := ExecuteDB(query)
 	if err != nil {
-		fmt.Println()
+		addNewLine()
 		Debugf("query: %s", query)
-		Fatalf("Error when updating the foreign key for table %s, err: %v", key.Table, err)
+		Errorf("Error when updating the foreign key for table %s, err: %v", key.Table, err)
 	}
 }
 
 // Delete the violating key
 func deleteViolatingConstraintKeys(tab string, column string) error {
-	Debugf("Deleting the rows of the table that violate the constraints: %s:(%s)", tab, column)
+	Debugf("Deleting the rows of the table that violate the constraints: %s(%s)", tab, column)
 	query := `
 DELETE 
 FROM   %[1]s 
@@ -570,6 +574,7 @@ WHERE  (
 	query = fmt.Sprintf(query, tab, column)
 	_, err := ExecuteDB(query)
 	if err != nil {
+		Debug("query: %s", query)
 		return err
 	}
 	return nil