CySQL Updates (#1005)

* feat: BED-5146 - support toString and toInt * feat: BED-5130 - support instant type components epochseconds and epocmillis for datetimes * fix: BED-5149, BED-5148 - implement cypher size function and rework type inference handling * fix: BED-5138 - apply dangling constraints to projection select for all shortest path queries * fix: BED-5130 - use numeric type casts to ensure that no loss of precision takes place
SpecterOps · Dec 10, 2024 · 288e098 · 288e098
1 parent d682a6a
commit 288e098
Show file tree

Hide file tree

Showing 18 changed files with 562 additions and 104 deletions.
diff --git a/cmd/api/src/api/v2/datapipe_integration_test.go b/cmd/api/src/api/v2/datapipe_integration_test.go
@@ -4,15 +4,15 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE2.0
+//     http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 //
-// SPDXLicenseIdentifier: Apache2.0
+// SPDX-License-Identifier: Apache-2.0
 
 //go:build serial_integration
 // +build serial_integration

diff --git a/packages/go/cypher/models/cypher/functions.go b/packages/go/cypher/models/cypher/functions.go
@@ -30,4 +30,21 @@ const (
 	NodeLabelsFunction         = "labels"
 	EdgeTypeFunction           = "type"
 	StringSplitToArrayFunction = "split"
+	ToStringFunction           = "tostring"
+	ToIntegerFunction          = "toint"
+	ListSizeFunction           = "size"
+
+	// ITTC - Instant Type; Temporal Component (https://neo4j.com/docs/cypher-manual/current/functions/temporal/)
+	ITTCYear              = "year"
+	ITTCMonth             = "month"
+	ITTCDay               = "day"
+	ITTCHour              = "hour"
+	ITTCMinute            = "minute"
+	ITTCSecond            = "second"
+	ITTCMillisecond       = "millisecond"
+	ITTCMicrosecond       = "microsecond"
+	ITTCNanosecond        = "nanosecond"
+	ITTCTimeZone          = "timezone"
+	ITTCEpochSeconds      = "epochseconds"
+	ITTCEpochMilliseconds = "epochmillis"
 )
diff --git a/packages/go/cypher/models/pgsql/format/format.go b/packages/go/cypher/models/pgsql/format/format.go
@@ -137,6 +137,12 @@ func formatValue(builder *OutputBuilder, value any) error {
 	case bool:
 		builder.Write(strconv.FormatBool(typedValue))
 
+	case float32:
+		builder.Write(strconv.FormatFloat(float64(typedValue), 'f', -1, 64))
+
+	case float64:
+		builder.Write(strconv.FormatFloat(typedValue, 'f', -1, 64))
+
 	default:
 		return fmt.Errorf("unsupported literal type: %T", value)
 	}
@@ -482,8 +488,27 @@ func formatNode(builder *OutputBuilder, rootExpr pgsql.SyntaxNode) error {
 				exprStack = append(exprStack, pgsql.FormattingLiteral("not "))
 			}
 
+		case pgsql.ProjectionFrom:
+			for idx, projection := range typedNextExpr.Projection {
+				if idx > 0 {
+					builder.Write(", ")
+				}
+
+				if err := formatNode(builder, projection); err != nil {
+					return err
+				}
+			}
+
+			if len(typedNextExpr.From) > 0 {
+				builder.Write(" from ")
+
+				if err := formatFromClauses(builder, typedNextExpr.From); err != nil {
+					return err
+				}
+			}
+
 		default:
-			return fmt.Errorf("unsupported node type: %T", nextExpr)
+			return fmt.Errorf("unable to format pgsql node type: %T", nextExpr)
 		}
 	}
 

diff --git a/packages/go/cypher/models/pgsql/functions.go b/packages/go/cypher/models/pgsql/functions.go
@@ -24,6 +24,7 @@ const (
 	FunctionJSONBToTextArray       Identifier = "jsonb_to_text_array"
 	FunctionJSONBArrayElementsText Identifier = "jsonb_array_elements_text"
 	FunctionJSONBBuildObject       Identifier = "jsonb_build_object"
+	FunctionJSONBArrayLength       Identifier = "jsonb_array_length"
 	FunctionArrayLength            Identifier = "array_length"
 	FunctionArrayAggregate         Identifier = "array_agg"
 	FunctionMin                    Identifier = "min"
@@ -41,4 +42,5 @@ const (
 	FunctionCount                  Identifier = "count"
 	FunctionStringToArray          Identifier = "string_to_array"
 	FunctionEdgesToPath            Identifier = "edges_to_path"
+	FunctionExtract                Identifier = "extract"
 )
diff --git a/packages/go/cypher/models/pgsql/identifiers.go b/packages/go/cypher/models/pgsql/identifiers.go
@@ -25,8 +25,23 @@ import (
 
 const (
 	WildcardIdentifier Identifier = "*"
+	EpochIdentifier    Identifier = "epoch"
 )
 
+var reservedIdentifiers = []Identifier{
+	EpochIdentifier,
+}
+
+func IsReservedIdentifier(identifier Identifier) bool {
+	for _, reservedIdentifier := range reservedIdentifiers {
+		if identifier == reservedIdentifier {
+			return true
+		}
+	}
+
+	return false
+}
+
 func AsOptionalIdentifier(identifier Identifier) models.Optional[Identifier] {
 	return models.ValueOptional(identifier)
 }

diff --git a/packages/go/cypher/models/pgsql/model.go b/packages/go/cypher/models/pgsql/model.go
@@ -403,9 +403,17 @@ type AnyExpression struct {
 }
 
 func NewAnyExpression(inner Expression) AnyExpression {
-	return AnyExpression{
+	newAnyExpression := AnyExpression{
 		Expression: inner,
 	}
+
+	// This is a guard to prevent recursive wrapping of an expression in an Any expression
+	switch innerTypeHint := inner.(type) {
+	case TypeHinted:
+		newAnyExpression.CastType = innerTypeHint.TypeHint()
+	}
+
+	return newAnyExpression
 }
 
 func (s AnyExpression) AsExpression() Expression {
@@ -972,6 +980,19 @@ func (s Projection) NodeType() string {
 	return "projection"
 }
 
+type ProjectionFrom struct {
+	Projection Projection
+	From       []FromClause
+}
+
+func (s ProjectionFrom) NodeType() string {
+	return "projection from"
+}
+
+func (s ProjectionFrom) AsExpression() Expression {
+	return s
+}
+
 // Select is a SQL expression that is evaluated to fetch data.
 type Select struct {
 	Distinct   bool

diff --git a/packages/go/cypher/models/pgsql/pgtypes.go b/packages/go/cypher/models/pgsql/pgtypes.go
@@ -93,6 +93,8 @@ const (
 	TextArray                DataType = "text[]"
 	JSONB                    DataType = "jsonb"
 	JSONBArray               DataType = "jsonb[]"
+	Numeric                  DataType = "numeric"
+	NumericArray             DataType = "numeric[]"
 	Date                     DataType = "date"
 	TimeWithTimeZone         DataType = "time with time zone"
 	TimeWithoutTimeZone      DataType = "time without time zone"
@@ -109,7 +111,8 @@ const (
 	ExpansionTerminalNode DataType = "expansion_terminal_node"
 )
 
-func (s DataType) Convert(other DataType) (DataType, bool) {
+// TODO: operator, while unused, is part of a refactor for this function to make it operator aware
+func (s DataType) Compatible(other DataType, operator Operator) (DataType, bool) {
 	if s == other {
 		return s, true
 	}
@@ -132,6 +135,12 @@ func (s DataType) Convert(other DataType) (DataType, bool) {
 		case Float8:
 			return Float8, true
 
+		case Float4Array:
+			return Float4, true
+
+		case Float8Array:
+			return Float8, true
+
 		case Text:
 			return Text, true
 		}
@@ -141,6 +150,21 @@ func (s DataType) Convert(other DataType) (DataType, bool) {
 		case Float4:
 			return Float8, true
 
+		case Float4Array, Float8Array:
+			return Float8, true
+
+		case Text:
+			return Text, true
+		}
+
+	case Numeric:
+		switch other {
+		case Float4, Float8, Int2, Int4, Int8:
+			return Numeric, true
+
+		case Float4Array, Float8Array, NumericArray:
+			return Numeric, true
+
 		case Text:
 			return Text, true
 		}
@@ -156,6 +180,15 @@ func (s DataType) Convert(other DataType) (DataType, bool) {
 		case Int8:
 			return Int8, true
 
+		case Int2Array:
+			return Int2, true
+
+		case Int4Array:
+			return Int4, true
+
+		case Int8Array:
+			return Int8, true
+
 		case Text:
 			return Text, true
 		}
@@ -168,6 +201,12 @@ func (s DataType) Convert(other DataType) (DataType, bool) {
 		case Int8:
 			return Int8, true
 
+		case Int2Array, Int4Array:
+			return Int4, true
+
+		case Int8Array:
+			return Int8, true
+
 		case Text:
 			return Text, true
 		}
@@ -177,9 +216,42 @@ func (s DataType) Convert(other DataType) (DataType, bool) {
 		case Int2, Int4, Int8:
 			return Int8, true
 
+		case Int2Array, Int4Array, Int8Array:
+			return Int8, true
+
+		case Text:
+			return Text, true
+		}
+
+	case Int:
+		switch other {
+		case Int2, Int4, Int:
+			return Int, true
+
+		case Int8:
+			return Int8, true
+
 		case Text:
 			return Text, true
 		}
+
+	case Int2Array:
+		switch other {
+		case Int2Array, Int4Array, Int8Array:
+			return other, true
+		}
+
+	case Int4Array:
+		switch other {
+		case Int4Array, Int8Array:
+			return other, true
+		}
+
+	case Float4Array:
+		switch other {
+		case Float4Array, Float8Array:
+			return other, true
+		}
 	}
 
 	return UnsetDataType, false
@@ -207,7 +279,7 @@ func (s DataType) MatchesOneOf(others ...DataType) bool {
 
 func (s DataType) IsArrayType() bool {
 	switch s {
-	case Int2Array, Int4Array, Int8Array, Float4Array, Float8Array, TextArray:
+	case Int2Array, Int4Array, Int8Array, Float4Array, Float8Array, TextArray, JSONBArray, NodeCompositeArray, EdgeCompositeArray, NumericArray:
 		return true
 	}
 
@@ -239,6 +311,8 @@ func (s DataType) ToArrayType() (DataType, error) {
 		return Float8Array, nil
 	case Text, TextArray:
 		return TextArray, nil
+	case Numeric, NumericArray:
+		return NumericArray, nil
 	default:
 		return UnknownDataType, ErrNoAvailableArrayDataType
 	}
@@ -258,8 +332,10 @@ func (s DataType) ArrayBaseType() (DataType, error) {
 		return Float8, nil
 	case TextArray:
 		return Text, nil
+	case NumericArray:
+		return Numeric, nil
 	default:
-		return UnknownDataType, ErrNonArrayDataType
+		return s, nil
 	}
 }
 

diff --git a/packages/go/cypher/models/pgsql/test/translation_cases/nodes.sql b/packages/go/cypher/models/pgsql/test/translation_cases/nodes.sql
@@ -82,7 +82,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from
      s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1
             from s0,
                  node n1
-            where (s0.n0).id = any (jsonb_to_text_array(n1.properties -> 'captured_ids')::int4[]))
+            where (s0.n0).id = any (jsonb_to_text_array(n1.properties -> 'captured_ids')::int8[]))
 select s1.n0 as s, s1.n1 as e
 from s1;
 
@@ -498,3 +498,66 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0
             where n0.properties ->> 'system_tags' like '%' || ('text')::text)
 select s0.n0 as n
 from s0;
+
+-- case: match (n:NodeKind1) where toString(n.functionallevel) in ['2008 R2','2012','2008','2003','2003 Interim','2000 Mixed/Native'] return n
+with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0
+            from node n0
+            where n0.kind_ids operator (pg_catalog.&&) array [1]::int2[]
+              and (n0.properties -> 'functionallevel')::text = any
+                  (array ['2008 R2', '2012', '2008', '2003', '2003 Interim', '2000 Mixed/Native']::text[]))
+select s0.n0 as n
+from s0;
+
+-- case: match (n:NodeKind1) where toInt(n.value) in [1, 2, 3, 4] return n
+with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0
+            from node n0
+            where n0.kind_ids operator (pg_catalog.&&) array [1]::int2[]
+              and (n0.properties -> 'value')::int8 = any (array [1, 2, 3, 4]::int8[]))
+select s0.n0 as n
+from s0;
+
+-- case: match (u:NodeKind1) where u.pwdlastset < (datetime().epochseconds - (365 * 86400)) and not u.pwdlastset IN [-1.0, 0.0] return u limit 100
+with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0
+            from node n0
+            where n0.kind_ids operator (pg_catalog.&&) array [1]::int2[]
+              and (n0.properties -> 'pwdlastset')::numeric <
+                  (extract(epoch from now()::timestamp with time zone)::numeric - (365 * 86400))
+              and not (n0.properties -> 'pwdlastset')::float8 = any (array [- 1, 0]::float8[]))
+select s0.n0 as u
+from s0
+limit 100;
+
+-- case: match (u:NodeKind1) where u.pwdlastset < (datetime().epochmillis - (365 * 86400000)) and not u.pwdlastset IN [-1.0, 0.0] return u limit 100
+with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0
+            from node n0
+            where n0.kind_ids operator (pg_catalog.&&) array [1]::int2[]
+              and (n0.properties -> 'pwdlastset')::numeric <
+                  (extract(epoch from now()::timestamp with time zone)::numeric * 1000 - (365 * 86400000))
+              and not (n0.properties -> 'pwdlastset')::float8 = any (array [- 1, 0]::float8[]))
+select s0.n0 as u
+from s0
+limit 100;
+
+-- case: match (n:NodeKind1) where size(n.array_value) > 0 return n
+with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0
+            from node n0
+            where n0.kind_ids operator (pg_catalog.&&) array [1]::int2[]
+              and jsonb_array_length(n0.properties -> 'array_value')::int > 0)
+select s0.n0 as n
+from s0;
+
+-- case: match (n) where 1 in n.array return n
+with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0
+            from node n0
+            where 1 = any (jsonb_to_text_array(n0.properties -> 'array')::int8[]))
+select s0.n0 as n
+from s0;
+
+-- case: match (n) where $p in n.array or $f in n.array return n
+-- cypher_params: {"p": 1, "f": "text"}
+with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0
+            from node n0
+            where @pi0::float8 = any (jsonb_to_text_array(n0.properties -> 'array')::float8[])
+               or @pi1::text = any (jsonb_to_text_array(n0.properties -> 'array')::text[]))
+select s0.n0 as n
+from s0;