expression, cmd: fix ColumnSubstitute and allow some cases to substit…

…ute (#38826)

close #38736, close #40536

cmd/explaintest/r/collation_check_use_collation_disabled.result

@@ -154,4 +154,28 @@ insert into t1 values ('-1');
 insert into t2 values (0x2d31, '');
 select * from t1, t2 where t1.a in (t2.b, 3);
 a	b	c
+drop table if exists t0;
+drop table if exists t1;
+CREATE TABLE t0(c0 BOOL, c1 INT);
+CREATE TABLE t1 LIKE t0;
+CREATE VIEW v0(c0) AS SELECT IS_IPV4(t0.c1) FROM t0, t1;
+INSERT INTO t0(c0, c1) VALUES (true, 0);
+INSERT INTO t1(c0, c1) VALUES (true, 2);
+SELECT v0.c0 FROM v0;
+c0
+0
+SELECT (v0.c0)NOT LIKE(BINARY v0.c0) FROM v0;
+(v0.c0)NOT LIKE(BINARY v0.c0)
+0
+SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);
+c0
+desc format='brief' SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);
+id	estRows	task	access object	operator info
+Projection	80000000.00	root		is_ipv4(cast(collation_check_use_collation.t0.c1, var_string(20)))->Column#7
+└─HashJoin	80000000.00	root		CARTESIAN inner join
+  ├─Selection(Build)	8000.00	root		not(like(cast(is_ipv4(cast(collation_check_use_collation.t0.c1, var_string(20))), var_string(20)), cast(is_ipv4(cast(collation_check_use_collation.t0.c1, var_string(20))), binary(1)), 92))
+  │ └─TableReader	10000.00	root		data:TableFullScan
+  │   └─TableFullScan	10000.00	cop[tikv]	table:t0	keep order:false, stats:pseudo
+  └─TableReader(Probe)	10000.00	root		data:TableFullScan
+    └─TableFullScan	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
 use test

cmd/explaintest/r/collation_check_use_collation_enabled.result

@@ -173,4 +173,28 @@ insert into t1 values ('-1');
 insert into t2 values (0x2d31, '');
 select * from t1, t2 where t1.a in (t2.b, 3);
 a	b	c
+drop table if exists t0;
+drop table if exists t1;
+CREATE TABLE t0(c0 BOOL, c1 INT);
+CREATE TABLE t1 LIKE t0;
+CREATE VIEW v0(c0) AS SELECT IS_IPV4(t0.c1) FROM t0, t1;
+INSERT INTO t0(c0, c1) VALUES (true, 0);
+INSERT INTO t1(c0, c1) VALUES (true, 2);
+SELECT v0.c0 FROM v0;
+c0
+0
+SELECT (v0.c0)NOT LIKE(BINARY v0.c0) FROM v0;
+(v0.c0)NOT LIKE(BINARY v0.c0)
+0
+SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);
+c0
+desc format='brief' SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);
+id	estRows	task	access object	operator info
+Projection	80000000.00	root		is_ipv4(cast(collation_check_use_collation.t0.c1, var_string(20)))->Column#7
+└─HashJoin	80000000.00	root		CARTESIAN inner join
+  ├─Selection(Build)	8000.00	root		not(like(cast(is_ipv4(cast(collation_check_use_collation.t0.c1, var_string(20))), var_string(20)), cast(is_ipv4(cast(collation_check_use_collation.t0.c1, var_string(20))), binary(1)), 92))
+  │ └─TableReader	10000.00	root		data:TableFullScan
+  │   └─TableFullScan	10000.00	cop[tikv]	table:t0	keep order:false, stats:pseudo
+  └─TableReader(Probe)	10000.00	root		data:TableFullScan
+    └─TableFullScan	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
 use test

cmd/explaintest/r/subquery.result

@@ -56,13 +56,11 @@ insert into exam values(1, 'math', 100);
 set names utf8 collate utf8_general_ci;
 explain format = 'brief' select * from stu where stu.name not in (select 'guo' from exam where exam.stu_id = stu.id);
 id	estRows	task	access object	operator info
-Apply	10000.00	root		CARTESIAN anti semi join, other cond:eq(test.stu.name, Column#8)
+HashJoin	8000.00	root		anti semi join, equal:[eq(test.stu.id, test.exam.stu_id)], other cond:eq(test.stu.name, "guo")
 ├─TableReader(Build)	10000.00	root		data:TableFullScan
-│ └─TableFullScan	10000.00	cop[tikv]	table:stu	keep order:false, stats:pseudo
-└─Projection(Probe)	100000.00	root		guo->Column#8
-  └─TableReader	100000.00	root		data:Selection
-    └─Selection	100000.00	cop[tikv]		eq(test.exam.stu_id, test.stu.id)
-      └─TableFullScan	100000000.00	cop[tikv]	table:exam	keep order:false, stats:pseudo
+│ └─TableFullScan	10000.00	cop[tikv]	table:exam	keep order:false, stats:pseudo
+└─TableReader(Probe)	10000.00	root		data:TableFullScan
+  └─TableFullScan	10000.00	cop[tikv]	table:stu	keep order:false, stats:pseudo
 select * from stu where stu.name not in (select 'guo' from exam where exam.stu_id = stu.id);
 id	name
 set names utf8mb4;

cmd/explaintest/t/collation_check_use_collation.test

@@ -110,5 +110,19 @@ insert into t1 values ('-1');
 insert into t2 values (0x2d31, '');
 select * from t1, t2 where t1.a in (t2.b, 3);
 
+# issue 38736
+drop table if exists t0;
+drop table if exists t1;
+CREATE TABLE t0(c0 BOOL, c1 INT);
+CREATE TABLE t1 LIKE t0;
+CREATE VIEW v0(c0) AS SELECT IS_IPV4(t0.c1) FROM t0, t1;
+INSERT INTO t0(c0, c1) VALUES (true, 0);
+INSERT INTO t1(c0, c1) VALUES (true, 2);
+
+SELECT v0.c0 FROM v0;
+SELECT (v0.c0)NOT LIKE(BINARY v0.c0) FROM v0;
+SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);
+desc format='brief' SELECT v0.c0 FROM v0 WHERE (v0.c0)NOT LIKE(BINARY v0.c0);
+
 # cleanup environment
 use test

expression/collation.go

@@ -299,14 +299,6 @@ func deriveCollation(ctx sessionctx.Context, funcName string, args []Expression,
 	return ec, nil
 }
 
-// DeriveCollationFromExprs derives collation information from these expressions.
-// Deprecated, use CheckAndDeriveCollationFromExprs instead.
-// TODO: remove this function after the all usage is replaced by CheckAndDeriveCollationFromExprs
-func DeriveCollationFromExprs(ctx sessionctx.Context, exprs ...Expression) (dstCharset, dstCollation string) {
-	collation := inferCollation(exprs...)
-	return collation.Charset, collation.Collation
-}
-
 // CheckAndDeriveCollationFromExprs derives collation information from these expressions, return error if derives collation error.
 func CheckAndDeriveCollationFromExprs(ctx sessionctx.Context, funcName string, evalType types.EvalType, args ...Expression) (et *ExprCollation, err error) {
 	ec := inferCollation(args...)

expression/integration_test.go

@@ -7893,3 +7893,12 @@ func TestIssue39146(t *testing.T) {
 	tk.MustExec("set @@tidb_enable_vectorized_expression = off;")
 	tk.MustQuery(`select str_to_date(substr(dest,1,6),'%H%i%s') from sun;`).Check(testkit.Rows("20:23:10"))
 }
+
+func TestIssue40536(t *testing.T) {
+	store := testkit.CreateMockStore(t)
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("use test")
+	tk.MustExec("CREATE TABLE `6bf9e76d-ab44-4031-8a07-418b10741580` (\n  `e0b5f703-6cfe-49b4-bc21-16a6455e43a7` set('7','va','ung60','ow','1g','gxwz5','uhnh','k','5la1','q8d9c','1f') NOT NULL DEFAULT '7,1g,uhnh,5la1,q8d9c',\n  `fbc3527f-9617-4b9d-a5dc-4be31c00d8a5` datetime DEFAULT '6449-09-28 14:39:04',\n  PRIMARY KEY (`e0b5f703-6cfe-49b4-bc21-16a6455e43a7`) /*T![clustered_index] CLUSTERED */\n) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_bin;")
+	tk.MustExec("CREATE TABLE `8919f3f4-25be-4a1a-904a-bb5e863d8fc8` (\n  `9804d5f2-cbc7-43b7-b241-ea2656dc941a` enum('s951','36d','ua65','49yru','6l2em','4ea','jf2d2','vprsc','3yl7n','hz','ov') DEFAULT '4ea',\n  `323cdbcb-0c14-4362-90ab-ea42caaed6a5` year(4) NOT NULL DEFAULT '1983',\n  `b9b70f39-1a02-4114-9d7d-fa6259c1b691` time DEFAULT '20:18:04',\n  PRIMARY KEY (`323cdbcb-0c14-4362-90ab-ea42caaed6a5`) /*T![clustered_index] CLUSTERED */,\n  KEY `a704d6bb-772b-44ea-8cb0-6f7491c1aaa6` (`323cdbcb-0c14-4362-90ab-ea42caaed6a5`,`9804d5f2-cbc7-43b7-b241-ea2656dc941a`)\n) ENGINE=InnoDB DEFAULT CHARSET=ascii COLLATE=ascii_bin;")
+	tk.MustExec("delete from `6bf9e76d-ab44-4031-8a07-418b10741580` where not( `6bf9e76d-ab44-4031-8a07-418b10741580`.`e0b5f703-6cfe-49b4-bc21-16a6455e43a7` in ( select `9804d5f2-cbc7-43b7-b241-ea2656dc941a` from `8919f3f4-25be-4a1a-904a-bb5e863d8fc8` where `6bf9e76d-ab44-4031-8a07-418b10741580`.`e0b5f703-6cfe-49b4-bc21-16a6455e43a7` in ( '1f' ) and `6bf9e76d-ab44-4031-8a07-418b10741580`.`e0b5f703-6cfe-49b4-bc21-16a6455e43a7` in ( '1g' ,'va' ,'uhnh' ) ) ) and not( IsNull( `6bf9e76d-ab44-4031-8a07-418b10741580`.`e0b5f703-6cfe-49b4-bc21-16a6455e43a7` ) );\n")
+}

expression/util.go

@@ -415,7 +415,6 @@ func ColumnSubstituteImpl(expr Expression, schema *Schema, newExprs []Expression
 		if v.InOperand {
 			newExpr = SetExprColumnInOperand(newExpr)
 		}
-		newExpr.SetCoercibility(v.Coercibility())
 		return true, false, newExpr
 	case *ScalarFunction:
 		substituted := false
@@ -438,7 +437,11 @@ func ColumnSubstituteImpl(expr Expression, schema *Schema, newExprs []Expression
 		// cowExprRef is a copy-on-write util, args array allocation happens only
 		// when expr in args is changed
 		refExprArr := cowExprRef{v.GetArgs(), nil}
-		_, coll := DeriveCollationFromExprs(v.GetCtx(), v.GetArgs()...)
+		oldCollEt, err := CheckAndDeriveCollationFromExprs(v.GetCtx(), v.FuncName.L, v.RetType.EvalType(), v.GetArgs()...)
+		if err != nil {
+			logutil.BgLogger().Error("Unexpected error happened during ColumnSubstitution", zap.Stack("stack"))
+			return false, false, v
+		}
 		var tmpArgForCollCheck []Expression
 		if collate.NewCollationEnabled() {
 			tmpArgForCollCheck = make([]Expression, len(v.GetArgs()))
@@ -454,9 +457,18 @@ func ColumnSubstituteImpl(expr Expression, schema *Schema, newExprs []Expression
 				changed = false
 				copy(tmpArgForCollCheck, refExprArr.Result())
 				tmpArgForCollCheck[idx] = newFuncExpr
-				_, newColl := DeriveCollationFromExprs(v.GetCtx(), tmpArgForCollCheck...)
-				if coll == newColl {
-					changed = checkCollationStrictness(coll, newFuncExpr.GetType().GetCollate())
+				newCollEt, err := CheckAndDeriveCollationFromExprs(v.GetCtx(), v.FuncName.L, v.RetType.EvalType(), tmpArgForCollCheck...)
+				if err != nil {
+					logutil.BgLogger().Error("Unexpected error happened during ColumnSubstitution", zap.Stack("stack"))
+					return false, failed, v
+				}
+				if oldCollEt.Collation == newCollEt.Collation {
+					if newFuncExpr.GetType().GetCollate() == arg.GetType().GetCollate() && newFuncExpr.Coercibility() == arg.Coercibility() {
+						// It's safe to use the new expression, otherwise some cases in projection push-down will be wrong.
+						changed = true
+					} else {
+						changed = checkCollationStrictness(oldCollEt.Collation, newFuncExpr.GetType().GetCollate())
+					}
 				}
 			}
 			hasFail = hasFail || failed || oldChanged != changed

planner/core/exhaust_physical_plans.go

@@ -1325,8 +1325,11 @@ func (ijHelper *indexJoinBuildHelper) resetContextForIndex(innerKeys []*expressi
 		if ijHelper.curIdxOff2KeyOff[i] >= 0 {
 			// Don't use the join columns if their collations are unmatched and the new collation is enabled.
 			if collate.NewCollationEnabled() && types.IsString(idxCol.RetType.GetType()) && types.IsString(outerKeys[ijHelper.curIdxOff2KeyOff[i]].RetType.GetType()) {
-				_, coll := expression.DeriveCollationFromExprs(nil, idxCol, outerKeys[ijHelper.curIdxOff2KeyOff[i]])
-				if !collate.CompatibleCollate(idxCol.GetType().GetCollate(), coll) {
+				et, err := expression.CheckAndDeriveCollationFromExprs(ijHelper.innerPlan.ctx, "equal", types.ETInt, idxCol, outerKeys[ijHelper.curIdxOff2KeyOff[i]])
+				if err != nil {
+					logutil.BgLogger().Error("Unexpected error happened during constructing index join", zap.Stack("stack"))
+				}
+				if !collate.CompatibleCollate(idxCol.GetType().GetCollate(), et.Collation) {
 					ijHelper.curIdxOff2KeyOff[i] = -1
 				}
 			}