Merge branch 'master' of git.postgrespro.ru:pgpro-dev/rum

Makefile

@@ -25,7 +25,7 @@ RELATIVE_INCLUDES = $(addprefix src/, $(INCLUDES))
 
 LDFLAGS_SL += $(filter -lm, $(LIBS))
 
-REGRESS = rum rum_validate rum_hash ruminv timestamp orderby orderby_hash \
+REGRESS = security rum rum_validate rum_hash ruminv timestamp orderby orderby_hash \
 	altorder altorder_hash limits \
 	int2 int4 int8 float4 float8 money oid \
     time timetz date interval \

expected/int4.out

@@ -145,7 +145,6 @@ SELECT id FROM test_int4_o WHERE  t @@ 'wr&qh' AND id >= 400 ORDER BY id;
 
 RESET enable_indexscan;
 RESET enable_indexonlyscan;
-RESET enable_bitmapscan;
 SET enable_seqscan = off;
 EXPLAIN (costs off)
 SELECT id, id <=> 400 FROM test_int4_o WHERE t @@ 'wr&qh' ORDER BY id <=> 400 LIMIT 5;
@@ -258,7 +257,6 @@ CREATE TABLE test_int4_a AS SELECT id::int4, t FROM tsts;
 CREATE INDEX test_int4_a_idx ON test_int4_a USING rum
 	(t rum_tsvector_addon_ops, id)
 	WITH (attach = 'id', to = 't', order_by_attach='t');
-SET enable_bitmapscan=OFF;
 EXPLAIN (costs off)
 SELECT count(*) FROM test_int4_a WHERE id < 400;
                       QUERY PLAN                       
@@ -448,7 +446,6 @@ SELECT id FROM test_int4_h_o WHERE  t @@ 'wr&qh' AND id >= 400 ORDER BY id;
 
 RESET enable_indexscan;
 RESET enable_indexonlyscan;
-RESET enable_bitmapscan;
 SET enable_seqscan = off;
 EXPLAIN (costs off)
 SELECT id, id <=> 400 FROM test_int4_h_o WHERE t @@ 'wr&qh' ORDER BY id <=> 400 LIMIT 5;
@@ -561,7 +558,6 @@ CREATE TABLE test_int4_h_a AS SELECT id::int4, t FROM tsts;
 CREATE INDEX test_int4_h_a_idx ON test_int4_h_a USING rum
 	(t rum_tsvector_hash_addon_ops, id)
 	WITH (attach = 'id', to = 't', order_by_attach='t');
-SET enable_bitmapscan=OFF;
 EXPLAIN (costs off)
 SELECT count(*) FROM test_int4_h_a WHERE id < 400;
                         QUERY PLAN                         

expected/int8.out

@@ -145,7 +145,6 @@ SELECT id FROM test_int8_o WHERE  t @@ 'wr&qh' AND id >= 400::int8 ORDER BY id;
 
 RESET enable_indexscan;
 RESET enable_indexonlyscan;
-RESET enable_bitmapscan;
 SET enable_seqscan = off;
 EXPLAIN (costs off)
 SELECT id, id <=> 400 FROM test_int8_o WHERE t @@ 'wr&qh' ORDER BY id <=> 400 LIMIT 5;
@@ -258,7 +257,6 @@ CREATE TABLE test_int8_a AS SELECT id::int8, t FROM tsts;
 CREATE INDEX test_int8_a_idx ON test_int8_a USING rum
 	(t rum_tsvector_addon_ops, id)
 	WITH (attach = 'id', to = 't', order_by_attach='t');
-SET enable_bitmapscan=OFF;
 EXPLAIN (costs off)
 SELECT count(*) FROM test_int8_a WHERE id < 400::int8;
                       QUERY PLAN                       
@@ -448,7 +446,6 @@ SELECT id FROM test_int8_h_o WHERE  t @@ 'wr&qh' AND id >= 400::int8 ORDER BY id
 
 RESET enable_indexscan;
 RESET enable_indexonlyscan;
-RESET enable_bitmapscan;
 SET enable_seqscan = off;
 EXPLAIN (costs off)
 SELECT id, id <=> 400 FROM test_int8_h_o WHERE t @@ 'wr&qh' ORDER BY id <=> 400 LIMIT 5;
@@ -561,7 +558,6 @@ CREATE TABLE test_int8_h_a AS SELECT id::int8, t FROM tsts;
 CREATE INDEX test_int8_h_a_idx ON test_int8_h_a USING rum
 	(t rum_tsvector_hash_addon_ops, id)
 	WITH (attach = 'id', to = 't', order_by_attach='t');
-SET enable_bitmapscan=OFF;
 EXPLAIN (costs off)
 SELECT count(*) FROM test_int8_h_a WHERE id < 400::int8;
                         QUERY PLAN                         

expected/int8_1.out

@@ -130,7 +130,6 @@ SELECT id FROM test_int8_o WHERE  t @@ 'wr&qh' AND id >= 400::int8 ORDER BY id;
 
 RESET enable_indexscan;
 RESET enable_indexonlyscan;
-RESET enable_bitmapscan;
 SET enable_seqscan = off;
 EXPLAIN (costs off)
 SELECT id, id <=> 400 FROM test_int8_o WHERE t @@ 'wr&qh' ORDER BY id <=> 400 LIMIT 5;
@@ -220,7 +219,6 @@ CREATE INDEX test_int8_a_idx ON test_int8_a USING rum
 	(t rum_tsvector_addon_ops, id)
 	WITH (attach = 'id', to = 't', order_by_attach='t');
 ERROR:  doesn't support order index over pass-by-reference column
-SET enable_bitmapscan=OFF;
 EXPLAIN (costs off)
 SELECT count(*) FROM test_int8_a WHERE id < 400::int8;
               QUERY PLAN              
@@ -413,7 +411,6 @@ SELECT id FROM test_int8_h_o WHERE  t @@ 'wr&qh' AND id >= 400::int8 ORDER BY id
 
 RESET enable_indexscan;
 RESET enable_indexonlyscan;
-RESET enable_bitmapscan;
 SET enable_seqscan = off;
 EXPLAIN (costs off)
 SELECT id, id <=> 400 FROM test_int8_h_o WHERE t @@ 'wr&qh' ORDER BY id <=> 400 LIMIT 5;
@@ -503,7 +500,6 @@ CREATE INDEX test_int8_h_a_idx ON test_int8_h_a USING rum
 	(t rum_tsvector_hash_addon_ops, id)
 	WITH (attach = 'id', to = 't', order_by_attach='t');
 ERROR:  doesn't support order index over pass-by-reference column
-SET enable_bitmapscan=OFF;
 EXPLAIN (costs off)
 SELECT count(*) FROM test_int8_h_a WHERE id < 400::int8;
               QUERY PLAN              

expected/rum.out

@@ -381,6 +381,24 @@ SELECT (a <=> to_tsquery('pg_catalog.english', 'b:*'))::numeric(10,4) AS distanc
   16.4493 | the few that escaped destruction in 1693. It is a beautiful, highly      | '1693':7 'beauti':11 'destruct':5 'escap':4 'high':12
 (20 rows)
 
+-- Test correct work of phrase operator when position information is not in index.
+create table test_rum_addon as table test_rum;
+alter table test_rum_addon add column id serial;
+create index on test_rum_addon using rum (a rum_tsvector_addon_ops, id) with (attach = 'id', to='a');
+select * from test_rum_addon where a @@  to_tsquery('pg_catalog.english', 'half <-> way');
+                                  t                                  |                            a                            | id 
+---------------------------------------------------------------------+---------------------------------------------------------+----
+ itself. Put on your "specs" and look at the castle, half way up the | 'castl':10 'half':11 'look':7 'put':2 'spec':5 'way':12 |  9
+(1 row)
+
+explain (costs off) select * from test_rum_addon where a @@  to_tsquery('pg_catalog.english', 'half <-> way');
+                         QUERY PLAN                         
+------------------------------------------------------------
+ Index Scan using test_rum_addon_a_id_idx on test_rum_addon
+   Index Cond: (a @@ '''half'' <-> ''way'''::tsquery)
+(2 rows)
+
+--
 select  ('bjarn:6237 stroustrup:6238'::tsvector <=> 'bjarn <-> stroustrup'::tsquery)::numeric(10,5) AS distance;
  distance 
 ----------

expected/security.out

@@ -0,0 +1,5 @@
+-- Check security CVE-2020-14350
+CREATE FUNCTION rum_anyarray_similar(anyarray,anyarray) RETURNS bool AS $$ SELECT false $$ LANGUAGE SQL;
+CREATE EXTENSION rum;
+ERROR:  function "rum_anyarray_similar" already exists with same argument types
+DROP FUNCTION rum_anyarray_similar(anyarray,anyarray);

gen_rum_sql--1.1--1.2.pl

@@ -82,7 +82,7 @@
 LANGUAGE C IMMUTABLE STRICT;
 
 
-CREATE OR REPLACE FUNCTION rum_anyarray_similar(anyarray,anyarray)
+CREATE FUNCTION rum_anyarray_similar(anyarray,anyarray)
 RETURNS bool
 AS 'MODULE_PATHNAME'
 LANGUAGE C STRICT STABLE;
@@ -97,7 +97,7 @@
 );
 
 
-CREATE OR REPLACE FUNCTION rum_anyarray_distance(anyarray,anyarray)
+CREATE FUNCTION rum_anyarray_distance(anyarray,anyarray)
 RETURNS float8
 AS 'MODULE_PATHNAME'
 LANGUAGE C STRICT STABLE;

rum--1.0.sql

@@ -1,4 +1,4 @@
-CREATE OR REPLACE FUNCTION rumhandler(internal)
+CREATE FUNCTION rumhandler(internal)
 RETURNS index_am_handler
 AS 'MODULE_PATHNAME'
 LANGUAGE C;

rum--1.1--1.2.sql

@@ -10,7 +10,7 @@ AS 'MODULE_PATHNAME'
 LANGUAGE C IMMUTABLE STRICT;
 
 
-CREATE OR REPLACE FUNCTION rum_anyarray_similar(anyarray,anyarray)
+CREATE FUNCTION rum_anyarray_similar(anyarray,anyarray)
 RETURNS bool
 AS 'MODULE_PATHNAME'
 LANGUAGE C STRICT STABLE;
@@ -25,7 +25,7 @@ CREATE OPERATOR % (
 );
 
 
-CREATE OR REPLACE FUNCTION rum_anyarray_distance(anyarray,anyarray)
+CREATE FUNCTION rum_anyarray_distance(anyarray,anyarray)
 RETURNS float8
 AS 'MODULE_PATHNAME'
 LANGUAGE C STRICT STABLE;

rum--1.1.sql

@@ -1,4 +1,4 @@
-CREATE OR REPLACE FUNCTION rumhandler(internal)
+CREATE FUNCTION rumhandler(internal)
 RETURNS index_am_handler
 AS 'MODULE_PATHNAME'
 LANGUAGE C;

rum--1.2.sql

@@ -1,4 +1,4 @@
-CREATE OR REPLACE FUNCTION rumhandler(internal)
+CREATE FUNCTION rumhandler(internal)
 RETURNS index_am_handler
 AS 'MODULE_PATHNAME'
 LANGUAGE C;
@@ -1527,7 +1527,7 @@ AS 'MODULE_PATHNAME'
 LANGUAGE C IMMUTABLE STRICT;
 
 
-CREATE OR REPLACE FUNCTION rum_anyarray_similar(anyarray,anyarray)
+CREATE FUNCTION rum_anyarray_similar(anyarray,anyarray)
 RETURNS bool
 AS 'MODULE_PATHNAME'
 LANGUAGE C STRICT STABLE;
@@ -1542,7 +1542,7 @@ CREATE OPERATOR % (
 );
 
 
-CREATE OR REPLACE FUNCTION rum_anyarray_distance(anyarray,anyarray)
+CREATE FUNCTION rum_anyarray_distance(anyarray,anyarray)
 RETURNS float8
 AS 'MODULE_PATHNAME'
 LANGUAGE C STRICT STABLE;

rum--1.3.sql

@@ -1,4 +1,4 @@
-CREATE OR REPLACE FUNCTION rumhandler(internal)
+CREATE FUNCTION rumhandler(internal)
 RETURNS index_am_handler
 AS 'MODULE_PATHNAME'
 LANGUAGE C;
@@ -1527,7 +1527,7 @@ AS 'MODULE_PATHNAME'
 LANGUAGE C IMMUTABLE STRICT;
 
 
-CREATE OR REPLACE FUNCTION rum_anyarray_similar(anyarray,anyarray)
+CREATE FUNCTION rum_anyarray_similar(anyarray,anyarray)
 RETURNS bool
 AS 'MODULE_PATHNAME'
 LANGUAGE C STRICT STABLE;
@@ -1542,7 +1542,7 @@ CREATE OPERATOR % (
 );
 
 
-CREATE OR REPLACE FUNCTION rum_anyarray_distance(anyarray,anyarray)
+CREATE FUNCTION rum_anyarray_distance(anyarray,anyarray)
 RETURNS float8
 AS 'MODULE_PATHNAME'
 LANGUAGE C STRICT STABLE;

sql/int4.sql

@@ -40,7 +40,6 @@ SELECT id FROM test_int4_o WHERE  t @@ 'wr&qh' AND id >= 400 ORDER BY id;
 
 RESET enable_indexscan;
 RESET enable_indexonlyscan;
-RESET enable_bitmapscan;
 SET enable_seqscan = off;
 
 EXPLAIN (costs off)
@@ -66,8 +65,6 @@ CREATE INDEX test_int4_a_idx ON test_int4_a USING rum
 	(t rum_tsvector_addon_ops, id)
 	WITH (attach = 'id', to = 't', order_by_attach='t');
 
-SET enable_bitmapscan=OFF;
-
 EXPLAIN (costs off)
 SELECT count(*) FROM test_int4_a WHERE id < 400;
 SELECT count(*) FROM test_int4_a WHERE id < 400;
@@ -107,7 +104,6 @@ SELECT id FROM test_int4_h_o WHERE  t @@ 'wr&qh' AND id >= 400 ORDER BY id;
 
 RESET enable_indexscan;
 RESET enable_indexonlyscan;
-RESET enable_bitmapscan;
 SET enable_seqscan = off;
 
 EXPLAIN (costs off)
@@ -133,8 +129,6 @@ CREATE INDEX test_int4_h_a_idx ON test_int4_h_a USING rum
 	(t rum_tsvector_hash_addon_ops, id)
 	WITH (attach = 'id', to = 't', order_by_attach='t');
 
-SET enable_bitmapscan=OFF;
-
 EXPLAIN (costs off)
 SELECT count(*) FROM test_int4_h_a WHERE id < 400;
 SELECT count(*) FROM test_int4_h_a WHERE id < 400;

sql/int8.sql

@@ -40,7 +40,6 @@ SELECT id FROM test_int8_o WHERE  t @@ 'wr&qh' AND id >= 400::int8 ORDER BY id;
 
 RESET enable_indexscan;
 RESET enable_indexonlyscan;
-RESET enable_bitmapscan;
 SET enable_seqscan = off;
 
 EXPLAIN (costs off)
@@ -66,8 +65,6 @@ CREATE INDEX test_int8_a_idx ON test_int8_a USING rum
 	(t rum_tsvector_addon_ops, id)
 	WITH (attach = 'id', to = 't', order_by_attach='t');
 
-SET enable_bitmapscan=OFF;
-
 EXPLAIN (costs off)
 SELECT count(*) FROM test_int8_a WHERE id < 400::int8;
 SELECT count(*) FROM test_int8_a WHERE id < 400::int8;
@@ -107,7 +104,6 @@ SELECT id FROM test_int8_h_o WHERE  t @@ 'wr&qh' AND id >= 400::int8 ORDER BY id
 
 RESET enable_indexscan;
 RESET enable_indexonlyscan;
-RESET enable_bitmapscan;
 SET enable_seqscan = off;
 
 EXPLAIN (costs off)
@@ -120,6 +116,7 @@ EXPLAIN (costs off)
 SELECT id, id |=> 400 FROM test_int8_h_o WHERE t @@ 'wr&qh' ORDER BY id |=> 400 LIMIT 5;
 SELECT id, id |=> 400 FROM test_int8_h_o WHERE t @@ 'wr&qh' ORDER BY id |=> 400 LIMIT 5;
 
+
 EXPLAIN (costs off)
 SELECT id FROM test_int8_h_o WHERE  t @@ 'wr&qh' AND id <= 400::int8 ORDER BY id;
 SELECT id FROM test_int8_h_o WHERE  t @@ 'wr&qh' AND id <= 400::int8 ORDER BY id;
@@ -133,8 +130,6 @@ CREATE INDEX test_int8_h_a_idx ON test_int8_h_a USING rum
 	(t rum_tsvector_hash_addon_ops, id)
 	WITH (attach = 'id', to = 't', order_by_attach='t');
 
-SET enable_bitmapscan=OFF;
-
 EXPLAIN (costs off)
 SELECT count(*) FROM test_int8_h_a WHERE id < 400::int8;
 SELECT count(*) FROM test_int8_h_a WHERE id < 400::int8;

sql/rum.sql

@@ -141,5 +141,14 @@ SELECT (a <=> to_tsquery('pg_catalog.english', 'b:*'))::numeric(10,4) AS distanc
 	WHERE a @@ to_tsquery('pg_catalog.english', 'b:*')
 	ORDER BY a <=> to_tsquery('pg_catalog.english', 'b:*');
 
+-- Test correct work of phrase operator when position information is not in index.
+create table test_rum_addon as table test_rum;
+alter table test_rum_addon add column id serial;
+create index on test_rum_addon using rum (a rum_tsvector_addon_ops, id) with (attach = 'id', to='a');
+
+select * from test_rum_addon where a @@  to_tsquery('pg_catalog.english', 'half <-> way');
+explain (costs off) select * from test_rum_addon where a @@  to_tsquery('pg_catalog.english', 'half <-> way');
+--
+
 select  ('bjarn:6237 stroustrup:6238'::tsvector <=> 'bjarn <-> stroustrup'::tsquery)::numeric(10,5) AS distance;
 SELECT  ('stroustrup:5508B,6233B,6238B bjarn:6235B,6237B' <=> 'bjarn <-> stroustrup'::tsquery)::numeric(10,5) AS distance;

sql/security.sql

@@ -0,0 +1,5 @@
+-- Check security CVE-2020-14350
+CREATE FUNCTION rum_anyarray_similar(anyarray,anyarray) RETURNS bool AS $$ SELECT false $$ LANGUAGE SQL;
+CREATE EXTENSION rum;
+DROP FUNCTION rum_anyarray_similar(anyarray,anyarray);
+

src/rum_ts_utils.c

@@ -244,19 +244,15 @@ rum_tsquery_pre_consistent(PG_FUNCTION_ARGS)
 		gcv.map_item_operand = (int *) (extra_data[0]);
 		gcv.need_recheck = &recheck;
 
-#if PG_VERSION_NUM >= 130000
-		res = TS_execute(GETQUERY(query),
-						 &gcv,
-						 TS_EXEC_PHRASE_NO_POS | TS_EXEC_SKIP_NOT,
-						 pre_checkcondition_rum);
-#else
 		res = TS_execute(GETQUERY(query),
 						 &gcv,
-						 TS_EXEC_PHRASE_NO_POS,
-						 pre_checkcondition_rum);
+						 TS_EXEC_PHRASE_NO_POS
+#if PG_VERSION_NUM >= 130000
+						 | TS_EXEC_SKIP_NOT
 #endif
+						 ,
+						 pre_checkcondition_rum);
 	}
-
 	PG_RETURN_BOOL(res);
 }
 
@@ -604,6 +600,14 @@ rum_phrase_execute(QueryItem *curitem, void *arg, uint32 flags,
 
 			if (curitem->qoperator.oper == OP_PHRASE)
 			{
+				/* In case of index where position is not available
+				 * (e.g. addon_ops) output TS_MAYBE even in case both
+				 * lmatch and rmatch are TS_YES. Otherwise we can lose
+				 * results of phrase queries.
+				 */
+				if (flags & TS_EXEC_PHRASE_NO_POS)
+					return TS_MAYBE;
+
 				/*
 				 * Compute Loffset and Roffset suitable for phrase match, and
 				 * compute overall width of whole phrase match.
@@ -840,6 +844,11 @@ rum_TS_execute(QueryItem *curitem, void *arg, uint32 flags,
 			 * converting at the topmost phrase operator gives results that
 			 * are bug-compatible with the old implementation, so do it like
 			 * this for now.
+			 *
+			 * Checking for TS_EXEC_PHRASE_NO_POS has been moved inside
+			 * rum_phrase_execute, otherwise we can lose results of phrase
+			 * operator when position information is not available in index
+			 * (e.g. index built with addon_ops)
 			 */
 			switch (rum_phrase_execute(curitem, arg, flags, chkcond, NULL))
 			{