From 6d64560cefa08df28b8da7fda5374b1e759f4305 Mon Sep 17 00:00:00 2001 From: Faisal Ali Date: Fri, 20 Mar 2020 20:28:55 +0000 Subject: [PATCH] Missing bracket (#35) * Fix for double brackets Fixes the issues related to brackets https://github.com/pivotal-gss/mock-data/issues/24 * Run all those constraints Fixes the issue https://github.com/pivotal-gss/mock-data/issues/31, This prevents from the program to fail when it has issue running constraint related queries. If there is a error executing a query to fix the constraint, throw the error on the screen and continue till the end. At the end the user can manually fix anything if there is any issue. * Bump the version number --- README.md | 9 +++--- constraintsRestore.go | 72 +++++++++++++++++++++++++++++++++---------- helpers.go | 24 +++++++++++++++ mock.go | 2 +- sql.go | 27 +++++++++------- 5 files changed, 101 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 383285c..e1de5dd 100644 --- a/README.md +++ b/README.md @@ -130,10 +130,11 @@ For more examples how to use the tool, please check out the [wiki](https://githu # Known Issues -1. If you have a composite unique index where one column is part of foreign key column then there are chances the constraint creation would fail. -2. Fixing CHECK constraints isn't supported due to complexity, so recreating check constraints would fail, use `custom` subcommand to control the data being inserted -3. On Greenplum Database partition tables are not supported (due to check constraint issues defined above), so use the `custom` sub command to define the data to be inserted to the column with check constraints -4. Custom data types are not supported, use `custom` sub command to control the data for that custom data types +1. We do struggle when recreating constraints, even though we do try to fix the primary key , foreign key, unique key. So there is no guarantee that the tool will fix all the constraints and manual intervention is needed in some cases. +2. If you have a composite unique index where one column is part of foreign key column then there are chances the constraint creation would fail. +3. Fixing CHECK constraints isn't supported due to complexity, so recreating check constraints would fail, use `custom` subcommand to control the data being inserted +4. On Greenplum Database partition tables are not supported (due to check constraint issues defined above), so use the `custom` sub command to define the data to be inserted to the column with check constraints +5. Custom data types are not supported, use `custom` sub command to control the data for that custom data types # Developers / Collaboration diff --git a/constraintsRestore.go b/constraintsRestore.go index 3e5d9c1..01170ae 100644 --- a/constraintsRestore.go +++ b/constraintsRestore.go @@ -9,6 +9,7 @@ var ( ignoreErr = []string{ "ERROR #42P16 multiple primary keys for table", "already exists"} + maxLoop = 10 ) // Get Foriegn key objects @@ -49,25 +50,25 @@ func FixConstraints() { func fixPKey(pk constraint) { Debugf("Fixing the Primary / Unique Key for table %s", pk.table) totalViolators := 1 - + totalLoop := 0 + // Extract the columns from the list that was collected during backup - keys, err := ColExtractor(pk.column, `\(.*?\)`) + keys, err := ColExtractor(pk.column, `\(([^\[\]]*)\)`) if err != nil { Fatalf("unable to determine PK violators key columns: %v", err) } - cols := strings.Trim(keys, "()") + cols := TrimPrefixNSuffix(RemoveEverySuffixAfterADelimiter(keys, " where "), "(", ")") - for totalViolators > 0 { // Loop till we get a 0 value (i.e 0 violation ) + for totalViolators > 0 && totalLoop <= maxLoop { // Loop till we get a 0 value (i.e 0 violation ) or max 10 loops // How many violations are we having, if zero then loop breaks totalViolators = getTotalPKViolator(pk.table, cols) if totalViolators > 0 { // Found violation, time to fix it // If there are two or more columns forming a PK or UK - // lets only fix column by column. - totalColumns := strings.Split(cols, ",") + pkColumns := strings.Split(cols, ",") // Get data type associated with the data types - dTypes := getDatatype(pk.table, totalColumns) + dTypes := getDatatype(pk.table, pkColumns) //Fix the primary constraints by picking the columns from the //array, i.e we update the column one by one. @@ -75,6 +76,9 @@ func fixPKey(pk constraint) { fixPKViolator(pk.table, v.Colname, v.Dtype) } } + // If there is still violation the function deleteViolatingPkOrUkConstraints takes + // care of it + totalLoop++ } } @@ -109,6 +113,15 @@ func fixFKey(con constraint) { Debugf("Checking / Fixing FOREIGN KEY Violation table: %s, column: %s, reference: %s(%s)", fkeyObjects.Table, fkeyObjects.Column, fkeyObjects.Reftable, fkeyObjects.Refcolumn) + // If its a composite FK relations then pick only one and fix it + // TODO: This is a bad logic, this will not quarantee a fix for the composite + // TODO: key. Clean this out later when get a proper solution to overcome + // TODO: the composite key + col := strings.Split(fkeyObjects.Column, ",") + refCol := strings.Split(fkeyObjects.Refcolumn, ",") + fkeyObjects.Column = col[0] + fkeyObjects.Refcolumn = refCol[0] + // Loop till we reach the the end of the loop for totalViolators > 0 { @@ -139,7 +152,7 @@ func getForeignKeyColumns(con constraint) *ForeignKey { if err != nil { Fatalf("Unable to extract foreign key column from fk clause: %v", err) } - fkCol = strings.Trim(fkCol, "()") + fkCol = TrimPrefixNSuffix(fkCol, "(", ")") // Extract the reference column from the clause refCol, err := ColExtractor(refClause, `\(.*?\)`) @@ -150,7 +163,7 @@ func getForeignKeyColumns(con constraint) *ForeignKey { // Extract reference table from the clause refTab := strings.Replace(refClause, refCol, "", -1) refTab = strings.Replace(refTab, "REFERENCES ", "", -1) - refCol = strings.Trim(refCol, "()") + refCol = TrimPrefixNSuffix(refCol, "(", ")") return &ForeignKey{con.table, fkCol, refTab, refCol} } @@ -187,7 +200,7 @@ func recreateAllConstraints() { } // Start the progress bar - bar := StartProgressBar(fmt.Sprintf("Recreated the Constraint Type \"%s\"", con), len(contents)) + bar := StartProgressBar(fmt.Sprintf("Recreating the constraint type \"%s\"", con), len(contents)) // Recreate all constraints one by one, if we can't create it then display the message // on the screen and continue with the rest, since we don't want it to fail if we cannot @@ -198,7 +211,7 @@ func recreateAllConstraints() { Debugf("Error creating constraint %s, err: %v", content, err) // Try an attempt to recreate constraint again after deleting the // violating row - successOrFailure := deleteViolatingPkOrUkConstriants(content) + successOrFailure := deleteViolatingPkOrUkConstraints(content) if !successOrFailure { // didn't succeed, ask the user to fix it manually err = WriteToFile(failedConstraintsFile, content+"\n") if err != nil { @@ -226,15 +239,13 @@ func recreateAllConstraints() { // is, we will delete the rows that violates it and hoping that it will help in // recreating the constraints. Yes we will loose that row at least that help to // recreate constraints ( fingers crossed :) ) -func deleteViolatingPkOrUkConstriants(con string) bool { +func deleteViolatingPkOrUkConstraints(con string) bool { Debugf("Attempting to run the constraint command %s second time, after deleting violating rows", con) // does the DDL contain PK or UK keyword then do the following // rest send them back for user to fix it. - if isSubStringAvailableOnString(con, "ADD CONSTRAINT.*PRIMARY KEY|ADD CONSTRAINT.*UNIQUE") { - column, _ := ColExtractor(con, `\(.*?\)`) - table, _ := ColExtractor(con, `ALTER TABLE(.*)ADD CONSTRAINT`) - table = strings.Trim(strings.Trim(table, "ALTER TABLE"), "ADD CONSTRAINT") - column = strings.Trim(column, "()") + if isSubStringAvailableOnString(con, "ADD CONSTRAINT.*PRIMARY KEY|ADD CONSTRAINT.*UNIQUE|CREATE UNIQUE INDEX") { + // Extract the table and column name + table, column := ExtractTableNColumnName(con) err := deleteViolatingConstraintKeys(table, column) if err != nil { // we failed to delete the the constraint violation rows Debugf("Error when deleting rows from the constraint violation rows: %v", err) @@ -250,3 +261,30 @@ func deleteViolatingPkOrUkConstriants(con string) bool { } return false } + +// Extract the table name and the column from the sql command +func ExtractTableNColumnName(s string) (string, string) { + var isItAlterStatement bool = true + var table string + var column string + + // Is this a create statement + if strings.HasPrefix(s, "CREATE UNIQUE") { // like create unique index statement + isItAlterStatement = false + } + + // Extract the column name + column, _ = ColExtractor(s, `\(([^\[\]]*)\)`) + column = TrimPrefixNSuffix(RemoveEverySuffixAfterADelimiter(column, " where "), "(", ")") + + // Extract the table name + switch isItAlterStatement { + case true: + table, _ = ColExtractor(s, `ALTER TABLE(.*)ADD CONSTRAINT`) + table = TrimPrefixNSuffix(table, "ALTER TABLE", "ADD CONSTRAINT") + case false: + table, _ = ColExtractor(s, `ON(.*)USING`) + table = TrimPrefixNSuffix(table, "ON", "USING") + } + return table, column +} diff --git a/helpers.go b/helpers.go index 7a5aae7..379faa2 100644 --- a/helpers.go +++ b/helpers.go @@ -206,6 +206,23 @@ func ColExtractor(conkey, regExp string) (string, error) { return "", nil } +// Trim brackets at the start and at the end +func TrimPrefixNSuffix(s, prefix, suffix string) string { + return strings.TrimPrefix(strings.TrimSuffix(s, suffix) , prefix) +} + +// Remove everything after a delimiter +func RemoveEverySuffixAfterADelimiter(s string, d string) string { + // Protect from upper case and lower case bugs + s = strings.ToLower(s) + d = strings.ToLower(d) + if strings.Contains(s, d) { + spiltString := strings.Split(s, d) + return spiltString[0] + } + return s +} + // If given a datatype see if it has a bracket or not. func BracketsExists(dt string) bool { var rgx = regexp.MustCompile(`\(.*\)`) @@ -263,4 +280,11 @@ func CharLen(dt string) (int, error) { returnValue = 1 } return returnValue, nil +} + +// New line if its not a debug +func addNewLine() { + if !cmdOptions.Debug { + fmt.Println() + } } \ No newline at end of file diff --git a/mock.go b/mock.go index ecc6287..8aca198 100644 --- a/mock.go +++ b/mock.go @@ -7,7 +7,7 @@ import ( var ( programName = "mock" - programVersion = "v2.5" + programVersion = "v2.6" ExecutionTimestamp = TimeNow() Path = fmt.Sprintf("%s/%s/%s", os.Getenv("HOME"), programName, ExecutionTimestamp) ) diff --git a/sql.go b/sql.go index 44cc754..3ffa5d3 100644 --- a/sql.go +++ b/sql.go @@ -412,8 +412,9 @@ func getTotalPKViolator(tab, cols string) int { _, err := db.Query(pg.Scan(&total), query) if err != nil { + addNewLine() Debugf("query: %s", query) - Fatalf("Error when executing the query to extract pk violators: %v", err) + Errorf("Error when executing the query to extract pk violators: %v", err) } return total @@ -436,8 +437,9 @@ func GetPKViolators(tab, cols string) []DBViolationRow { query := strings.Replace(getPKViolator(tab, cols), "SELECT "+cols, "SELECT "+cols+" AS row", -1) _, err := db.Query(&result, query) if err != nil { + addNewLine() Debugf("query: %s", query) - Fatalf("Error when executing the query to extract pk violators for table %s: %v", tab, err) + Errorf("Error when executing the query to extract pk violators for table %s: %v", tab, err) } return result @@ -457,9 +459,9 @@ WHERE ctid = query = fmt.Sprintf(query, tab, col, newdata, whichrow) _, err := ExecuteDB(query) if err != nil { - fmt.Println() + addNewLine() Debugf("query: %s", query) - Fatalf("Error when updating the primary key for table %s, err: %v", tab, err) + Errorf("Error when updating the primary key for table %s, err: %v", tab, err) } return "" } @@ -489,8 +491,9 @@ func GetTotalFKViolators(key ForeignKey) int { _, err := db.Query(pg.Scan(&total), query) if err != nil { + addNewLine() Debugf("Query: %s", query) - Fatalf("Error when executing the query to total rows of foreign keys for table %s: %v", key.Table, err) + Errorf("Error when executing the query to total rows of foreign keys for table %s: %v", key.Table, err) } return total @@ -507,9 +510,9 @@ func TotalRows(tab string) int { _, err := db.Query(pg.Scan(&total), query) if err != nil { - fmt.Println() + addNewLine() Debugf("query: %s", query) - Fatalf("Error when executing the query to total rows: %v", err) + Errorf("Error when executing the query to total rows: %v", err) } return total @@ -527,8 +530,9 @@ func GetFKViolators(key ForeignKey) []DBViolationRow { query := strings.Replace(getFKViolators(key), "SELECT "+key.Column, "SELECT "+key.Column+" AS row", -1) _, err := db.Query(&result, query) if err != nil { + addNewLine() Debugf("query: %s", query) - Fatalf("Error when executing the query to extract fk violators for table %s: %v", key.Table, err) + Errorf("Error when executing the query to extract fk violators for table %s: %v", key.Table, err) } return result @@ -547,15 +551,15 @@ WHERE %[2]s = '%[6]s' query = fmt.Sprintf(query, key.Table, key.Column, key.Refcolumn, key.Reftable, totalRows, whichRow) _, err := ExecuteDB(query) if err != nil { - fmt.Println() + addNewLine() Debugf("query: %s", query) - Fatalf("Error when updating the foreign key for table %s, err: %v", key.Table, err) + Errorf("Error when updating the foreign key for table %s, err: %v", key.Table, err) } } // Delete the violating key func deleteViolatingConstraintKeys(tab string, column string) error { - Debugf("Deleting the rows of the table that violate the constraints: %s:(%s)", tab, column) + Debugf("Deleting the rows of the table that violate the constraints: %s(%s)", tab, column) query := ` DELETE FROM %[1]s @@ -570,6 +574,7 @@ WHERE ( query = fmt.Sprintf(query, tab, column) _, err := ExecuteDB(query) if err != nil { + Debug("query: %s", query) return err } return nil