Add support for WE8MSWIN1252 character set when using a oracle db

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/urfave/negroni v1.0.0
 	golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85
 	golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35 // indirect
+	golang.org/x/text v0.3.0
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/goracle.v2 v2.10.2
 )

internal/pkg/sql/oracle/oracle.go

@@ -16,20 +16,26 @@ import (
 	"github.com/signavio/workflow-connector/internal/pkg/log"
 	sqlBackend "github.com/signavio/workflow-connector/internal/pkg/sql"
 	"github.com/signavio/workflow-connector/internal/pkg/util"
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/charmap"
+	"golang.org/x/text/encoding/unicode"
 	"gopkg.in/goracle.v2"
 )
 
 type lastId struct {
 	id int64
 }
+type characterSet encoding.Encoding
 
 const (
 	dateTimeOracleFormat = `'YYYY-MM-DD"T"HH24:MI:SSXFF3TZH:TZM'`
 	dateTimeGolangFormat = `2006-01-02T15:04:05.999-07:00`
 )
 
 var (
-	QueryTemplates = map[string]string{
+	Universal         characterSet = unicode.UTF8
+	EuroSymbolSupport characterSet = charmap.Windows1252
+	QueryTemplates                 = map[string]string{
 		`GetSingle`: `SELECT * ` +
 			`FROM {{.TableName}} "_{{.TableName}}"` +
 			`{{range .Relations}}` +
@@ -116,6 +122,7 @@ var (
 
 type Oracle struct {
 	*sqlBackend.SqlBackend
+	characterSet characterSet
 }
 
 func (l *lastId) LastInsertId() (int64, error) {
@@ -126,7 +133,8 @@ func (l *lastId) RowsAffected() (int64, error) {
 	return 0, nil
 }
 func New() endpoint.Endpoint {
-	o := &Oracle{sqlBackend.New().(*sqlBackend.SqlBackend)}
+	// Assume UTF-8 character set before checking
+	o := &Oracle{sqlBackend.New().(*sqlBackend.SqlBackend), Universal}
 	o.Templates = QueryTemplates
 	o.ExecContextFunc = wrapExecContext(o.DB, o.ExecContextFunc)
 	o.CastDatabaseTypeToGolangType = convertFromOracleDataType
@@ -141,6 +149,26 @@ func driverSpecificInitialization(ctx context.Context, db *sql.DB) error {
 	}
 	return nil
 }
+func (o *Oracle) setCharacterSet() (err error) {
+	getCharacterSet :=
+		`SELECT VALUE FROM NLS_DATABASE_PARAMETERS WHERE PARAMETER = 'NLS_CHARACTERSET'`
+	var charSet string
+	err = o.DB.QueryRowContext(context.Background(), getCharacterSet).Scan(&charSet)
+	if err != nil {
+		log.When(config.Options.Logging).Infof("Error retrieving current character encoding from db: %s", err)
+		return fmt.Errorf("Error retrieving current character encoding from db: %s", err)
+	}
+	switch charSet {
+	case "AL32UTF8":
+		o.characterSet = Universal
+	case "WE8MSWIN1252":
+		o.characterSet = EuroSymbolSupport
+	default:
+		// Unsupported character set
+		return fmt.Errorf("Character set '%s' is not supported", charSet)
+	}
+	return nil
+}
 func (o *Oracle) Open(args ...interface{}) error {
 	log.When(config.Options.Logging).Infof(
 		"[backend] open connection to database %v\n",
@@ -161,12 +189,51 @@ func (o *Oracle) Open(args ...interface{}) error {
 		log.When(config.Options.Logging).Infof("Error performing driver specific initialization: %s", err)
 		return fmt.Errorf("Error performing driver specific initialization: %s", err)
 	}
+	if err = o.setCharacterSet(); err != nil {
+		return err
+	}
+	o.QueryContextFunc = wrapQueryContext(o.characterSet, o.QueryContextFunc)
 	err = o.SaveSchemaMapping()
 	if err != nil {
 		return fmt.Errorf("Error saving table schema: %s", err)
 	}
 	return nil
 }
+func wrapQueryContext(charSet characterSet, queryContext func(context.Context, string, ...interface{}) ([]interface{}, error)) func(context.Context, string, ...interface{}) ([]interface{}, error) {
+	return func(ctx context.Context, query string, args ...interface{}) ([]interface{}, error) {
+		var resultsAsUtf8 []interface{}
+		results, err := queryContext(ctx, query, args...)
+		for _, result := range results {
+			resultAsUtf8, err := convertCharacterSetToUtf8(charSet, result)
+			if err != nil {
+				return nil, err
+			}
+			resultsAsUtf8 = append(resultsAsUtf8, resultAsUtf8)
+		}
+		return resultsAsUtf8, err
+	}
+}
+func convertCharacterSetToUtf8(charSet characterSet, queryResult interface{}) (interface{}, error) {
+	var err error
+	utf8ResultOuter := make(map[string]interface{})
+	tableAndRelationships := queryResult.(map[string]interface{})
+	for ki, _ := range tableAndRelationships {
+		utf8ResultInner := make(map[string]interface{})
+		for kj, vj := range tableAndRelationships[ki].(map[string]interface{}) {
+			switch vj.(type) {
+			case string:
+				utf8ResultInner[kj], err = charSet.NewEncoder().String(vj.(string))
+				if err != nil {
+					return nil, err
+				}
+			default:
+				utf8ResultInner[kj] = vj
+			}
+		}
+		utf8ResultOuter[ki] = utf8ResultInner
+	}
+	return utf8ResultOuter, nil
+}
 func wrapExecContext(db *sql.DB, execContext func(context.Context, string, ...interface{}) (sql.Result, error)) func(context.Context, string, ...interface{}) (sql.Result, error) {
 	return func(ctx context.Context, query string, args ...interface{}) (sql.Result, error) {
 		lastInserted := bytes.NewBufferString("")