Merge pull request #1392 from hyperledger/vent-no-blob

[Vent] Add BytesToHex flag to byte32 fields to hex string columns

CHANGELOG.md

@@ -1,8 +1,13 @@
 # [Hyperledger Burrow](https://github.com/hyperledger/burrow) Changelog
+## [0.30.5] - 2020-07-09
+### Added
+- [Vent] Add BytesToHex flag on projection field mappings that causes bytes fields (e.g. bytes32) solidity fields to be hex-encoded and mapped to varchar(64) rather than bytea/blob columns in postgres/sqlite
+
+
 ## [0.30.4] - 2020-04-05
 ### Added
 - [Build] Added Helm chart
-- [State] Account now has OpcodeBitset field to support upcoming EVM fixes
+- [State] Account now has EVMOpcodeBitset field to support upcoming EVM fixes
 
 ### Fixed
 - [JS] Github actions release of JS lib
@@ -661,6 +666,7 @@ This release marks the start of Eris-DB as the full permissioned blockchain node
   - [Blockchain] Fix getBlocks to respect block height cap.
 
 
+[0.30.5]: https://github.com/hyperledger/burrow/compare/v0.30.4...v0.30.5
 [0.30.4]: https://github.com/hyperledger/burrow/compare/v0.30.3...v0.30.4
 [0.30.3]: https://github.com/hyperledger/burrow/compare/v0.30.2...v0.30.3
 [0.30.2]: https://github.com/hyperledger/burrow/compare/v0.30.1...v0.30.2

NOTES.md

@@ -1,7 +1,3 @@
 ### Added
-- [Build] Added Helm chart
-- [State] Account now has OpcodeBitset field to support upcoming EVM fixes
-
-### Fixed
-- [JS] Github actions release of JS lib
+- [Vent] Add BytesToHex flag on projection field mappings that causes bytes fields (e.g. bytes32) solidity fields to be hex-encoded and mapped to varchar(64) rather than bytea/blob columns in postgres/sqlite
 

docker-compose.yml

@@ -6,8 +6,6 @@ services:
       - POSTGRES_HOST_AUTH_METHOD=trust
     ports:
       - 5432
-    environment:
-      - POSTGRES_HOST_AUTH_METHOD=trust
 
   burrow:
     build: .github

project/history.go

@@ -47,7 +47,12 @@ func FullVersion() string {
 // To cut a new release add a release to the front of this slice then run the
 // release tagging script: ./scripts/tag_release.sh
 var History relic.ImmutableHistory = relic.NewHistory("Hyperledger Burrow", "https://github.com/hyperledger/burrow").
-	MustDeclareReleases("0.30.4 - 2020-04-05",
+	MustDeclareReleases(
+		"0.30.5 - 2020-07-09",
+		`### Added
+- [Vent] Add BytesToHex flag on projection field mappings that causes bytes fields (e.g. bytes32) solidity fields to be hex-encoded and mapped to varchar(64) rather than bytea/blob columns in postgres/sqlite
+`,
+		"0.30.4 - 2020-04-05",
 		`### Added
 - [Build] Added Helm chart
 - [State] Account now has EVMOpcodeBitset field to support upcoming EVM fixes

vent/service/consumer_postgres_test.go

@@ -111,8 +111,9 @@ func TestPostgresConsumer(t *testing.T) {
 				t.Logf("latest height: %d, txe height: %d", height, txe.Height)
 				assert.True(t, height >= txe.Height)
 			}
-			assert.Equal(t, `{"_action" : "INSERT", "testdescription" : "\\x5472696767657220697421000000000000000000000000000000000000000000", "testname" : "TestTriggerEvent"}`, notifications["meta"])
-			assert.Equal(t, `{"_action" : "INSERT", "testdescription" : "\\x5472696767657220697421000000000000000000000000000000000000000000", "testkey" : "\\x544553545f4556454e5453000000000000000000000000000000000000000000", "testname" : "TestTriggerEvent"}`,
+			assert.Equal(t, `{"_action" : "INSERT", "testdescription" : "5472696767657220697421000000000000000000000000000000000000000000", "testname" : "TestTriggerEvent"}`,
+				notifications["meta"])
+			assert.Equal(t, `{"_action" : "INSERT", "testdescription" : "5472696767657220697421000000000000000000000000000000000000000000", "testkey" : "\\x544553545f4556454e5453000000000000000000000000000000000000000000", "testname" : "TestTriggerEvent"}`,
 				notifications["keyed_meta"])
 		})
 	})

vent/service/rowbuilder.go

@@ -13,6 +13,7 @@ import (
 	"github.com/hyperledger/burrow/vent/sqlsol"
 	"github.com/hyperledger/burrow/vent/types"
 	"github.com/pkg/errors"
+	"github.com/tmthrgd/go-hex"
 )
 
 // buildEventData builds event data from transactions
@@ -50,11 +51,12 @@ func buildEventData(projection *sqlsol.Projection, eventClass *types.EventClass,
 		}
 		column, err := projection.GetColumn(eventClass.TableName, fieldMapping.ColumnName)
 		if err == nil {
-			if fieldMapping.BytesToString {
-				if bs, ok := value.(*[]byte); ok {
+			if bs, ok := value.(*[]byte); ok {
+				if fieldMapping.BytesToString {
 					str := sanitiseBytesForString(*bs, logger)
-					row[column.Name] = interface{}(str)
-					continue
+					value = interface{}(str)
+				} else if fieldMapping.BytesToHex {
+					value = hex.EncodeUpperToString(*bs)
 				}
 			}
 			row[column.Name] = value

vent/sqlsol/projection.go

@@ -120,7 +120,13 @@ func NewProjection(spec types.ProjectionSpec) (*Projection, error) {
 
 		i := 0
 		for _, mapping := range eventClass.FieldMappings {
-			sqlType, sqlTypeLength, err := getSQLType(mapping.Type, mapping.BytesToString)
+			var bytesMapping BytesMapping
+			if mapping.BytesToHex {
+				bytesMapping = BytesToHex
+			} else if mapping.BytesToString {
+				bytesMapping = BytesToString
+			}
+			sqlType, sqlTypeLength, err := getSQLType(mapping.Type, bytesMapping)
 			if err != nil {
 				return nil, err
 			}
@@ -220,9 +226,17 @@ func readFile(file string) ([]byte, error) {
 	return byteValue, nil
 }
 
+type BytesMapping int
+
+const (
+	BytesToBytes = iota
+	BytesToString
+	BytesToHex
+)
+
 // getSQLType maps event input types with corresponding SQL column types
 // takes into account related solidity types info and element indexed or hashed
-func getSQLType(evmSignature string, bytesToString bool) (types.SQLColumnType, int, error) {
+func getSQLType(evmSignature string, bytesMapping BytesMapping) (types.SQLColumnType, int, error) {
 	evmSignature = strings.ToLower(evmSignature)
 	re := regexp.MustCompile("[0-9]+")
 	typeSize, _ := strconv.Atoi(re.FindString(evmSignature))
@@ -237,9 +251,12 @@ func getSQLType(evmSignature string, bytesToString bool) (types.SQLColumnType, i
 		// solidity bytes => sql bytes
 		// bytesToString == true means there is a string in there so => sql varchar
 	case strings.HasPrefix(evmSignature, types.EventFieldTypeBytes):
-		if bytesToString {
-			return types.SQLColumnTypeVarchar, 40, nil
-		} else {
+		switch bytesMapping {
+		case BytesToString:
+			return types.SQLColumnTypeVarchar, 32, nil
+		case BytesToHex:
+			return types.SQLColumnTypeVarchar, 64, nil
+		default:
 			return types.SQLColumnTypeByteA, 0, nil
 		}
 		// solidity string => sql text

vent/test/sqlsol.go

@@ -109,7 +109,7 @@ func UnknownTypeJSONConfFile(t *testing.T) string {
 			},
 			"Fields"  , 
 				"ColumnName" , "ColumnName" : "testname", "Primary" : true},
-				"description", "ColumnName" : "testdescription", "Primary" : false}
+				"description", "ColumnName" : "testdescription", "Primary" : false, "BytesToHex": true}
 			}
 		}
 	]`

vent/test/sqlsol_log.json

@@ -23,6 +23,7 @@
         "ColumnName": "testdescription",
         "Type": "bytes32",
         "Primary": false,
+        "BytesToHex": true,
         "Notify": ["meta", "keyed_meta"]
       }
     ]

vent/test/sqlsol_view.json

@@ -24,6 +24,7 @@
         "ColumnName": "testdescription",
         "Type": "bytes32",
         "Primary": false,
+        "BytesToHex": true,
         "Notify": ["meta", "keyed_meta"]
       }
     ]

vent/types/event_class.go

@@ -79,6 +79,8 @@ type EventFieldMapping struct {
 	Primary bool `json:",omitempty"`
 	// Whether to convert this event field from bytes32 to string
 	BytesToString bool `json:",omitempty"`
+	// Whether to convert this event field from bytes32 to hex-encoded string
+	BytesToHex bool `json:",omitempty"`
 	// Notification channels on which submit (via a trigger) a payload that contains this column's new value (upsert) or
 	// old value (delete). The payload will contain all other values with the same channel set as a JSON object.
 	Notify []string `json:",omitempty"`