diff --git a/src/common/expression/LabelAttributeExpression.h b/src/common/expression/LabelAttributeExpression.h index 3ed02581f01..40cee0d7fb0 100644 --- a/src/common/expression/LabelAttributeExpression.h +++ b/src/common/expression/LabelAttributeExpression.h @@ -34,7 +34,7 @@ class LabelAttributeExpression final : public Expression { } const Value& eval(ExpressionContext&) override { - LOG(FATAL) << "LabelAttributeExpression has to be rewritten"; + DLOG(FATAL) << "LabelAttributeExpression has to be rewritten"; return Value::kNullBadData; } diff --git a/src/graph/context/Iterator.cpp b/src/graph/context/Iterator.cpp index 80f4f7d1845..637caa61712 100644 --- a/src/graph/context/Iterator.cpp +++ b/src/graph/context/Iterator.cpp @@ -440,7 +440,7 @@ Value GetNeighborsIter::getVertex(const std::string& name) const { return Value::kNullValue; } - auto vidVal = getColumn(nebula::kVid); + auto vidVal = getColumn(0); if (UNLIKELY(!SchemaUtil::isValidVid(vidVal))) { return Value::kNullBadType; } @@ -502,12 +502,13 @@ Value GetNeighborsIter::getEdge() const { edge.name = edgeName; auto type = getEdgeProp(edgeName, kType); - if (!type.isInt()) { - return Value::kNullBadType; + if (type.isInt()) { + edge.type = type.getInt(); + } else { + edge.type = 0; } - edge.type = type.getInt(); - auto& srcVal = getColumn(kVid); + auto& srcVal = getColumn(0); if (!SchemaUtil::isValidVid(srcVal)) { return Value::kNullBadType; } @@ -520,10 +521,11 @@ Value GetNeighborsIter::getEdge() const { edge.dst = dstVal; auto& rank = getEdgeProp(edgeName, kRank); - if (!rank.isInt()) { - return Value::kNullBadType; + if (rank.isInt()) { + edge.ranking = rank.getInt(); + } else { + edge.ranking = 0; } - edge.ranking = rank.getInt(); auto& edgePropMap = currentDs_->edgePropsMap; auto edgeProp = edgePropMap.find(currentEdgeName()); @@ -535,7 +537,7 @@ Value GetNeighborsIter::getEdge() const { DCHECK_EQ(edgeNamePropList.size(), propList.size()); for (size_t i = 0; i < propList.size(); ++i) { auto propName = edgeNamePropList[i]; - if (propName == kSrc || propName == kDst || propName == kRank || propName == kType) { + if (propName == kDst || propName == kRank || propName == kType || propName == kSrc) { continue; } edge.props.emplace(edgeNamePropList[i], propList[i]); diff --git a/src/graph/context/Result.h b/src/graph/context/Result.h index a23d701cd6f..bf52ac70e6f 100644 --- a/src/graph/context/Result.h +++ b/src/graph/context/Result.h @@ -63,6 +63,15 @@ class Result final { } } + std::vector getColNames() const { + auto& ds = value(); + if (ds.isDataSet()) { + return ds.getDataSet().colNames; + } + + return {}; + } + private: friend class ResultBuilder; friend class ExecutionContext; diff --git a/src/graph/executor/Executor.cpp b/src/graph/executor/Executor.cpp index 70290e4b5b4..4807248c382 100644 --- a/src/graph/executor/Executor.cpp +++ b/src/graph/executor/Executor.cpp @@ -695,6 +695,10 @@ bool Executor::movable(const Variable *var) { return false; } if (node()->loopLayers() != 0) { + // Guaranteed forward compatibility of go statement execution behavior + if (node()->kind() == PlanNode::Kind::kFilter) { + return true; + } // The lifetime of loop body is managed by Loop node return false; } diff --git a/src/graph/executor/StorageAccessExecutor.cpp b/src/graph/executor/StorageAccessExecutor.cpp index 48fcdb58d3f..86b100a6076 100644 --- a/src/graph/executor/StorageAccessExecutor.cpp +++ b/src/graph/executor/StorageAccessExecutor.cpp @@ -37,11 +37,12 @@ struct Vid { }; template -DataSet buildRequestDataSet(const SpaceInfo &space, - QueryExpressionContext &exprCtx, - Iterator *iter, - Expression *expr, - bool dedup) { +StatusOr buildRequestDataSet(const SpaceInfo &space, + QueryExpressionContext &exprCtx, + Iterator *iter, + Expression *expr, + bool dedup, + bool isCypher) { DCHECK(iter && expr) << "iter=" << iter << ", expr=" << expr; nebula::DataSet vertices({kVid}); auto s = iter->size(); @@ -54,11 +55,19 @@ DataSet buildRequestDataSet(const SpaceInfo &space, for (; iter->valid(); iter->next()) { auto vid = expr->eval(exprCtx(iter)); - if (!SchemaUtil::isValidVid(vid, vidType)) { - LOG(WARNING) << "Mismatched vid type: " << vid.type() - << ", space vid type: " << SchemaUtil::typeToString(vidType); + if (vid.empty()) { continue; } + if (!SchemaUtil::isValidVid(vid, vidType)) { + if (isCypher) { + continue; + } + std::stringstream ss; + ss << "`" << vid.toString() << "', the srcs should be type of " + << apache::thrift::util::enumNameSafe(vidType.get_type()) << ", but was`" << vid.type() + << "'"; + return Status::Error(ss.str()); + } if (dedup && !uniqueSet.emplace(Vid::value(vid)).second) { continue; } @@ -73,16 +82,17 @@ bool StorageAccessExecutor::isIntVidType(const SpaceInfo &space) const { return (*space.spaceDesc.vid_type_ref()).type == nebula::cpp2::PropertyType::INT64; } -DataSet StorageAccessExecutor::buildRequestDataSetByVidType(Iterator *iter, - Expression *expr, - bool dedup) { +StatusOr StorageAccessExecutor::buildRequestDataSetByVidType(Iterator *iter, + Expression *expr, + bool dedup, + bool isCypher) { const auto &space = qctx()->rctx()->session()->space(); QueryExpressionContext exprCtx(qctx()->ectx()); if (isIntVidType(space)) { - return internal::buildRequestDataSet(space, exprCtx, iter, expr, dedup); + return internal::buildRequestDataSet(space, exprCtx, iter, expr, dedup, isCypher); } - return internal::buildRequestDataSet(space, exprCtx, iter, expr, dedup); + return internal::buildRequestDataSet(space, exprCtx, iter, expr, dedup, isCypher); } std::string StorageAccessExecutor::getStorageDetail( diff --git a/src/graph/executor/StorageAccessExecutor.h b/src/graph/executor/StorageAccessExecutor.h index 0978ca5bba3..0e18c9333bb 100644 --- a/src/graph/executor/StorageAccessExecutor.h +++ b/src/graph/executor/StorageAccessExecutor.h @@ -158,7 +158,10 @@ class StorageAccessExecutor : public Executor { bool isIntVidType(const SpaceInfo &space) const; - DataSet buildRequestDataSetByVidType(Iterator *iter, Expression *expr, bool dedup); + StatusOr buildRequestDataSetByVidType(Iterator *iter, + Expression *expr, + bool dedup, + bool isCypher = false); }; } // namespace graph diff --git a/src/graph/executor/query/AppendVerticesExecutor.cpp b/src/graph/executor/query/AppendVerticesExecutor.cpp index d64a470cbe8..d4fe327e763 100644 --- a/src/graph/executor/query/AppendVerticesExecutor.cpp +++ b/src/graph/executor/query/AppendVerticesExecutor.cpp @@ -9,28 +9,32 @@ using nebula::storage::StorageClient; using nebula::storage::StorageRpcResponse; using nebula::storage::cpp2::GetPropResponse; - +DECLARE_bool(optimize_appendvertices); namespace nebula { namespace graph { folly::Future AppendVerticesExecutor::execute() { return appendVertices(); } -DataSet AppendVerticesExecutor::buildRequestDataSet(const AppendVertices *av) { +StatusOr AppendVerticesExecutor::buildRequestDataSet(const AppendVertices *av) { if (av == nullptr) { return nebula::DataSet({kVid}); } auto valueIter = ectx_->getResult(av->inputVar()).iter(); - return buildRequestDataSetByVidType(valueIter.get(), av->src(), av->dedup()); + return buildRequestDataSetByVidType(valueIter.get(), av->src(), av->dedup(), true); } folly::Future AppendVerticesExecutor::appendVertices() { SCOPED_TIMER(&execTime_); - auto *av = asNode(node()); - StorageClient *storageClient = qctx()->getStorageClient(); + if (FLAGS_optimize_appendvertices && av != nullptr && av->props() == nullptr) { + return handleNullProp(av); + } - DataSet vertices = buildRequestDataSet(av); + StorageClient *storageClient = qctx()->getStorageClient(); + auto res = buildRequestDataSet(av); + NG_RETURN_IF_ERROR(res); + auto vertices = std::move(res).value(); if (vertices.rows.empty()) { return finish(ResultBuilder().value(Value(DataSet(av->colNames()))).build()); } @@ -67,6 +71,39 @@ folly::Future AppendVerticesExecutor::appendVertices() { }); } +Status AppendVerticesExecutor::handleNullProp(const AppendVertices *av) { + auto iter = ectx_->getResult(av->inputVar()).iter(); + auto *src = av->src(); + + auto size = iter->size(); + DataSet ds; + ds.colNames = av->colNames(); + ds.rows.reserve(size); + + QueryExpressionContext ctx(ectx_); + bool canBeMoved = movable(av->inputVars().front()); + + for (; iter->valid(); iter->next()) { + const auto &vid = src->eval(ctx(iter.get())); + if (vid.empty()) { + continue; + } + Vertex vertex; + vertex.vid = vid; + if (!av->trackPrevPath()) { + Row row; + row.values.emplace_back(std::move(vertex)); + ds.rows.emplace_back(std::move(row)); + } else { + Row row; + row = canBeMoved ? iter->moveRow() : *iter->row(); + row.values.emplace_back(std::move(vertex)); + ds.rows.emplace_back(std::move(row)); + } + } + return finish(ResultBuilder().value(Value(std::move(ds))).build()); +} + Status AppendVerticesExecutor::handleResp( storage::StorageRpcResponse &&rpcResp) { auto result = handleCompleteness(rpcResp, FLAGS_accept_partial_success); diff --git a/src/graph/executor/query/AppendVerticesExecutor.h b/src/graph/executor/query/AppendVerticesExecutor.h index bbe3d697d54..88ce56870ac 100644 --- a/src/graph/executor/query/AppendVerticesExecutor.h +++ b/src/graph/executor/query/AppendVerticesExecutor.h @@ -22,12 +22,14 @@ class AppendVerticesExecutor final : public GetPropExecutor { folly::Future execute() override; private: - DataSet buildRequestDataSet(const AppendVertices *gv); + StatusOr buildRequestDataSet(const AppendVertices *gv); folly::Future appendVertices(); Status handleResp(storage::StorageRpcResponse &&rpcResp); + Status handleNullProp(const AppendVertices *av); + folly::Future handleRespMultiJobs( storage::StorageRpcResponse &&rpcResp); diff --git a/src/graph/executor/query/FilterExecutor.cpp b/src/graph/executor/query/FilterExecutor.cpp index b824472a4a2..65d4d8382cc 100644 --- a/src/graph/executor/query/FilterExecutor.cpp +++ b/src/graph/executor/query/FilterExecutor.cpp @@ -78,32 +78,57 @@ StatusOr FilterExecutor::handleJob(size_t begin, size_t end, Iterator * Status FilterExecutor::handleSingleJobFilter() { auto *filter = asNode(node()); - Result result = ectx_->getResult(filter->inputVar()); + auto inputVar = filter->inputVar(); + // Use the userCount of the operator's inputVar at runtime to determine whether concurrent + // read-write conflicts exist, and if so, copy the data + bool canMoveData = movable(inputVar); + Result result = ectx_->getResult(inputVar); auto *iter = result.iterRef(); - + // Always reuse getNeighbors's dataset to avoid some go statement execution plan related issues + if (iter->isGetNeighborsIter()) { + canMoveData = true; + } ResultBuilder builder; - builder.value(result.valuePtr()); QueryExpressionContext ctx(ectx_); auto condition = filter->condition(); - while (iter->valid()) { - auto val = condition->eval(ctx(iter)); - if (val.isBadNull() || (!val.empty() && !val.isImplicitBool() && !val.isNull())) { - return Status::Error("Wrong type result, the type should be NULL, EMPTY, BOOL"); - } - if (val.empty() || val.isNull() || (val.isImplicitBool() && !val.implicitBool())) { - if (UNLIKELY(filter->needStableFilter())) { - iter->erase(); + if (LIKELY(canMoveData)) { + builder.value(result.valuePtr()); + while (iter->valid()) { + auto val = condition->eval(ctx(iter)); + if (val.isBadNull() || (!val.empty() && !val.isImplicitBool() && !val.isNull())) { + return Status::Error("Wrong type result, the type should be NULL, EMPTY, BOOL"); + } + if (val.empty() || val.isNull() || (val.isImplicitBool() && !val.implicitBool())) { + if (UNLIKELY(filter->needStableFilter())) { + iter->erase(); + } else { + iter->unstableErase(); + } } else { - iter->unstableErase(); + iter->next(); } - } else { - iter->next(); } - } - iter->reset(); - builder.iter(std::move(result).iter()); - return finish(builder.build()); + iter->reset(); + builder.iter(std::move(result).iter()); + return finish(builder.build()); + } else { + DataSet ds; + ds.colNames = result.getColNames(); + ds.rows.reserve(iter->size()); + for (; iter->valid(); iter->next()) { + auto val = condition->eval(ctx(iter)); + if (val.isBadNull() || (!val.empty() && !val.isImplicitBool() && !val.isNull())) { + return Status::Error("Wrong type result, the type should be NULL, EMPTY, BOOL"); + } + if (val.isImplicitBool() && val.implicitBool()) { + Row row; + row = *iter->row(); + ds.rows.emplace_back(std::move(row)); + } + } + return finish(builder.value(Value(std::move(ds))).iter(Iterator::Kind::kProp).build()); + } } } // namespace graph diff --git a/src/graph/executor/query/GetEdgesExecutor.cpp b/src/graph/executor/query/GetEdgesExecutor.cpp index a129df1d3d2..8e248df8b9b 100644 --- a/src/graph/executor/query/GetEdgesExecutor.cpp +++ b/src/graph/executor/query/GetEdgesExecutor.cpp @@ -5,6 +5,7 @@ #include "graph/executor/query/GetEdgesExecutor.h" #include "graph/planner/plan/Query.h" +#include "graph/util/SchemaUtil.h" using nebula::storage::StorageClient; using nebula::storage::StorageRpcResponse; @@ -17,7 +18,7 @@ folly::Future GetEdgesExecutor::execute() { return getEdges(); } -DataSet GetEdgesExecutor::buildRequestDataSet(const GetEdges *ge) { +StatusOr GetEdgesExecutor::buildRequestDataSet(const GetEdges *ge) { auto valueIter = ectx_->getResult(ge->inputVar()).iter(); QueryExpressionContext exprCtx(qctx()->ectx()); @@ -25,11 +26,28 @@ DataSet GetEdgesExecutor::buildRequestDataSet(const GetEdges *ge) { edges.rows.reserve(valueIter->size()); std::unordered_set> uniqueEdges; uniqueEdges.reserve(valueIter->size()); + + const auto &space = qctx()->rctx()->session()->space(); + const auto &vidType = *(space.spaceDesc.vid_type_ref()); for (; valueIter->valid(); valueIter->next()) { auto type = ge->type()->eval(exprCtx(valueIter.get())); auto src = ge->src()->eval(exprCtx(valueIter.get())); auto dst = ge->dst()->eval(exprCtx(valueIter.get())); auto rank = ge->ranking()->eval(exprCtx(valueIter.get())); + if (!SchemaUtil::isValidVid(src, vidType)) { + std::stringstream ss; + ss << "`" << src.toString() << "', the src should be type of " + << apache::thrift::util::enumNameSafe(vidType.get_type()) << ", but was`" << src.type() + << "'"; + return Status::Error(ss.str()); + } + if (!SchemaUtil::isValidVid(dst, vidType)) { + std::stringstream ss; + ss << "`" << dst.toString() << "', the dst should be type of " + << apache::thrift::util::enumNameSafe(vidType.get_type()) << ", but was`" << dst.type() + << "'"; + return Status::Error(ss.str()); + } type = type < 0 ? -type : type; auto edgeKey = std::make_tuple(src, type, rank, dst); if (ge->dedup() && !uniqueEdges.emplace(std::move(edgeKey)).second) { @@ -52,7 +70,9 @@ folly::Future GetEdgesExecutor::getEdges() { return Status::Error("ptr is nullptr"); } - auto edges = buildRequestDataSet(ge); + auto res = buildRequestDataSet(ge); + NG_RETURN_IF_ERROR(res); + auto edges = std::move(res).value(); if (edges.rows.empty()) { // TODO: add test for empty input. diff --git a/src/graph/executor/query/GetEdgesExecutor.h b/src/graph/executor/query/GetEdgesExecutor.h index 39802c21f43..20446d87d23 100644 --- a/src/graph/executor/query/GetEdgesExecutor.h +++ b/src/graph/executor/query/GetEdgesExecutor.h @@ -19,7 +19,7 @@ class GetEdgesExecutor final : public GetPropExecutor { folly::Future execute() override; private: - DataSet buildRequestDataSet(const GetEdges *ge); + StatusOr buildRequestDataSet(const GetEdges *ge); folly::Future getEdges(); }; diff --git a/src/graph/executor/query/GetNeighborsExecutor.cpp b/src/graph/executor/query/GetNeighborsExecutor.cpp index 9c90e66c578..3737f741088 100644 --- a/src/graph/executor/query/GetNeighborsExecutor.cpp +++ b/src/graph/executor/query/GetNeighborsExecutor.cpp @@ -13,7 +13,7 @@ using nebula::storage::cpp2::GetNeighborsResponse; namespace nebula { namespace graph { -DataSet GetNeighborsExecutor::buildRequestDataSet() { +StatusOr GetNeighborsExecutor::buildRequestDataSet() { SCOPED_TIMER(&execTime_); auto inputVar = gn_->inputVar(); auto iter = ectx_->getResult(inputVar).iter(); @@ -21,7 +21,9 @@ DataSet GetNeighborsExecutor::buildRequestDataSet() { } folly::Future GetNeighborsExecutor::execute() { - DataSet reqDs = buildRequestDataSet(); + auto res = buildRequestDataSet(); + NG_RETURN_IF_ERROR(res); + auto reqDs = std::move(res).value(); if (reqDs.rows.empty()) { List emptyResult; return finish(ResultBuilder() diff --git a/src/graph/executor/query/GetNeighborsExecutor.h b/src/graph/executor/query/GetNeighborsExecutor.h index e7d79786e59..7ba20c19cf5 100644 --- a/src/graph/executor/query/GetNeighborsExecutor.h +++ b/src/graph/executor/query/GetNeighborsExecutor.h @@ -21,7 +21,7 @@ class GetNeighborsExecutor final : public StorageAccessExecutor { folly::Future execute() override; - DataSet buildRequestDataSet(); + StatusOr buildRequestDataSet(); private: using RpcResponse = storage::StorageRpcResponse; diff --git a/src/graph/executor/query/GetVerticesExecutor.cpp b/src/graph/executor/query/GetVerticesExecutor.cpp index 6a71c737e67..20014b74090 100644 --- a/src/graph/executor/query/GetVerticesExecutor.cpp +++ b/src/graph/executor/query/GetVerticesExecutor.cpp @@ -21,7 +21,9 @@ folly::Future GetVerticesExecutor::getVertices() { auto *gv = asNode(node()); StorageClient *storageClient = qctx()->getStorageClient(); - DataSet vertices = buildRequestDataSet(gv); + auto res = buildRequestDataSet(gv); + NG_RETURN_IF_ERROR(res); + auto vertices = std::move(res).value(); if (vertices.rows.empty()) { // TODO: add test for empty input. return finish( @@ -55,7 +57,7 @@ folly::Future GetVerticesExecutor::getVertices() { }); } -DataSet GetVerticesExecutor::buildRequestDataSet(const GetVertices *gv) { +StatusOr GetVerticesExecutor::buildRequestDataSet(const GetVertices *gv) { if (gv == nullptr) { return nebula::DataSet({kVid}); } diff --git a/src/graph/executor/query/GetVerticesExecutor.h b/src/graph/executor/query/GetVerticesExecutor.h index 24d2c249039..d52ff39adb8 100644 --- a/src/graph/executor/query/GetVerticesExecutor.h +++ b/src/graph/executor/query/GetVerticesExecutor.h @@ -19,7 +19,7 @@ class GetVerticesExecutor final : public GetPropExecutor { folly::Future execute() override; private: - DataSet buildRequestDataSet(const GetVertices *gv); + StatusOr buildRequestDataSet(const GetVertices *gv); folly::Future getVertices(); }; diff --git a/src/graph/executor/query/LimitExecutor.cpp b/src/graph/executor/query/LimitExecutor.cpp index 38502d2bf3b..236d8fe3cf6 100644 --- a/src/graph/executor/query/LimitExecutor.cpp +++ b/src/graph/executor/query/LimitExecutor.cpp @@ -13,16 +13,37 @@ folly::Future LimitExecutor::execute() { SCOPED_TIMER(&execTime_); auto* limit = asNode(node()); - Result result = ectx_->getResult(limit->inputVar()); + auto inputVar = limit->inputVar(); + // Use the userCount of the operator's inputVar at runtime to determine whether concurrent + // read-write conflicts exist, and if so, copy the data + bool canMoveData = movable(inputVar); + Result result = ectx_->getResult(inputVar); auto* iter = result.iterRef(); + // Always reuse getNeighbors's dataset to avoid some go statement execution plan related issues + if (iter->isGetNeighborsIter()) { + canMoveData = true; + } ResultBuilder builder; - builder.value(result.valuePtr()); - auto offset = limit->offset(); + auto offset = static_cast(limit->offset()); QueryExpressionContext qec(ectx_); - auto count = limit->count(qec); - iter->select(offset, count); - builder.iter(std::move(result).iter()); - return finish(builder.build()); + auto count = static_cast(limit->count(qec)); + if (LIKELY(canMoveData)) { + builder.value(result.valuePtr()); + iter->select(offset, count); + builder.iter(std::move(result).iter()); + return finish(builder.build()); + } else { + DataSet ds; + ds.colNames = result.getColNames(); + for (std::size_t i = 0; iter->valid(); ++i, iter->next()) { + if (i >= offset && i <= (offset + count - 1)) { + Row row; + row = *iter->row(); + ds.rows.emplace_back(std::move(row)); + } + } + return finish(builder.value(Value(std::move(ds))).iter(Iterator::Kind::kProp).build()); + } } } // namespace graph diff --git a/src/graph/executor/query/ScanEdgesExecutor.cpp b/src/graph/executor/query/ScanEdgesExecutor.cpp index de354a8748f..ed2d8945e00 100644 --- a/src/graph/executor/query/ScanEdgesExecutor.cpp +++ b/src/graph/executor/query/ScanEdgesExecutor.cpp @@ -40,10 +40,10 @@ folly::Future ScanEdgesExecutor::scanEdges() { SCOPED_TIMER(&execTime_); otherStats_.emplace("total_rpc", folly::sformat("{}(us)", scanEdgesTime.elapsedInUSec())); }) - .thenValue([this, se](StorageRpcResponse &&rpcResp) { + .thenValue([this](StorageRpcResponse &&rpcResp) { SCOPED_TIMER(&execTime_); addStats(rpcResp, otherStats_); - return handleResp(std::move(rpcResp), se->colNames()); + return handleResp(std::move(rpcResp), {}); }); } diff --git a/src/graph/executor/query/UnwindExecutor.cpp b/src/graph/executor/query/UnwindExecutor.cpp index de1d774e0a7..1e42993fa3b 100644 --- a/src/graph/executor/query/UnwindExecutor.cpp +++ b/src/graph/executor/query/UnwindExecutor.cpp @@ -26,7 +26,7 @@ folly::Future UnwindExecutor::execute() { std::vector vals = extractList(list); for (auto &v : vals) { Row row; - if (!emptyInput) { + if (!unwind->fromPipe() && !emptyInput) { row = *(iter->row()); } row.values.emplace_back(std::move(v)); diff --git a/src/graph/executor/test/GetNeighborsTest.cpp b/src/graph/executor/test/GetNeighborsTest.cpp index 9edf1352e05..55c3ba5e48c 100644 --- a/src/graph/executor/test/GetNeighborsTest.cpp +++ b/src/graph/executor/test/GetNeighborsTest.cpp @@ -69,7 +69,8 @@ TEST_F(GetNeighborsTest, BuildRequestDataSet) { gn->setInputVar("input_gn"); auto gnExe = std::make_unique(gn, qctx_.get()); - auto reqDs = gnExe->buildRequestDataSet(); + auto res = gnExe->buildRequestDataSet(); + auto reqDs = std::move(res).value(); DataSet expected; expected.colNames = {kVid}; diff --git a/src/graph/optimizer/rule/GetEdgesTransformUtils.h b/src/graph/optimizer/rule/GetEdgesTransformUtils.h index 15c5f98fac3..75a05a1a8fd 100644 --- a/src/graph/optimizer/rule/GetEdgesTransformUtils.h +++ b/src/graph/optimizer/rule/GetEdgesTransformUtils.h @@ -48,6 +48,9 @@ class GetEdgesTransformUtils final { limit_count, {}, traverse->filter() == nullptr ? nullptr : traverse->filter()->clone()); + auto &colNames = traverse->colNames(); + DCHECK_EQ(colNames.size(), 2); + scanEdges->setColNames({colNames.back()}); return scanEdges; } diff --git a/src/graph/optimizer/rule/OptimizeEdgeIndexScanByFilterRule.cpp b/src/graph/optimizer/rule/OptimizeEdgeIndexScanByFilterRule.cpp index f02f025d5e7..8b4fb01aa39 100644 --- a/src/graph/optimizer/rule/OptimizeEdgeIndexScanByFilterRule.cpp +++ b/src/graph/optimizer/rule/OptimizeEdgeIndexScanByFilterRule.cpp @@ -118,7 +118,7 @@ StatusOr OptimizeEdgeIndexScanByFilterRule::transform( if (static_cast(inExpr->right())->size() > 1) { return TransformResult::noTransform(); } else { - transformedExpr = graph::ExpressionUtils::rewriteInExpr(condition); + transformedExpr = graph::ExpressionUtils::rewriteInExpr(inExpr); } if (OptimizerUtils::relExprHasIndex(inExpr, indexItems)) { return TransformResult::noTransform(); diff --git a/src/graph/planner/SequentialPlanner.cpp b/src/graph/planner/SequentialPlanner.cpp index f7095d5fec6..08350e57d44 100644 --- a/src/graph/planner/SequentialPlanner.cpp +++ b/src/graph/planner/SequentialPlanner.cpp @@ -19,10 +19,8 @@ bool SequentialPlanner::match(AstContext* astCtx) { StatusOr SequentialPlanner::transform(AstContext* astCtx) { SubPlan subPlan; auto* seqCtx = static_cast(astCtx); - auto* qctx = seqCtx->qctx; const auto& validators = seqCtx->validators; subPlan.root = validators.back()->root(); - ifBuildDataCollect(subPlan, qctx); for (auto iter = validators.begin(); iter < validators.end() - 1; ++iter) { // Remove left tail kStart plannode before append plan. // It allows that kUse sentence to append kMatch Sentence. @@ -40,30 +38,6 @@ StatusOr SequentialPlanner::transform(AstContext* astCtx) { return subPlan; } -void SequentialPlanner::ifBuildDataCollect(SubPlan& subPlan, QueryContext* qctx) { - switch (subPlan.root->kind()) { - case PlanNode::Kind::kSort: - case PlanNode::Kind::kLimit: - case PlanNode::Kind::kSample: - case PlanNode::Kind::kDedup: - case PlanNode::Kind::kUnion: - case PlanNode::Kind::kUnionAllVersionVar: - case PlanNode::Kind::kIntersect: - case PlanNode::Kind::kCartesianProduct: - case PlanNode::Kind::kMinus: - case PlanNode::Kind::kFilter: { - auto* dc = DataCollect::make(qctx, DataCollect::DCKind::kRowBasedMove); - dc->addDep(subPlan.root); - dc->setInputVars({subPlan.root->outputVar()}); - dc->setColNames(subPlan.root->colNames()); - subPlan.root = dc; - break; - } - default: - break; - } -} - // When appending plans, it need to remove left tail plannode. // Because the left tail plannode is StartNode which needs to be removed, // and remain one size for add dependency diff --git a/src/graph/planner/SequentialPlanner.h b/src/graph/planner/SequentialPlanner.h index a37efdf7e97..769e49d8cdc 100644 --- a/src/graph/planner/SequentialPlanner.h +++ b/src/graph/planner/SequentialPlanner.h @@ -26,8 +26,6 @@ class SequentialPlanner final : public Planner { */ StatusOr transform(AstContext* astCtx) override; - void ifBuildDataCollect(SubPlan& subPlan, QueryContext* qctx); - void rmLeftTailStartNode(Validator* validator, Sentence::Kind appendPlanKind); private: diff --git a/src/graph/planner/match/MatchSolver.cpp b/src/graph/planner/match/MatchSolver.cpp index b1c5b95c6f1..1aa4e66ba6e 100644 --- a/src/graph/planner/match/MatchSolver.cpp +++ b/src/graph/planner/match/MatchSolver.cpp @@ -213,62 +213,6 @@ Expression* MatchSolver::makeIndexFilter(const std::string& label, return root; } -void MatchSolver::extractAndDedupVidColumn(QueryContext* qctx, - Expression** initialExpr, - PlanNode* dep, - const std::string& inputVar, - SubPlan& plan) { - auto columns = qctx->objPool()->makeAndAdd(); - auto* var = qctx->symTable()->getVar(inputVar); - Expression* vidExpr = initialExprOrEdgeDstExpr(qctx, initialExpr, var->colNames.back()); - if (initialExpr) { - *initialExpr = nullptr; - } - columns->addColumn(new YieldColumn(vidExpr)); - auto project = Project::make(qctx, dep, columns); - project->setInputVar(inputVar); - project->setColNames({kVid}); - auto dedup = Dedup::make(qctx, project); - dedup->setColNames({kVid}); - - plan.root = dedup; -} - -Expression* MatchSolver::initialExprOrEdgeDstExpr(QueryContext* qctx, - Expression** initialExpr, - const std::string& vidCol) { - if (initialExpr != nullptr && *initialExpr != nullptr) { - return *initialExpr; - } else { - return getEndVidInPath(qctx, vidCol); - } -} - -Expression* MatchSolver::getEndVidInPath(QueryContext* qctx, const std::string& colName) { - auto* pool = qctx->objPool(); - // expr: __Project_2[-1] => path - auto columnExpr = InputPropertyExpression::make(pool, colName); - // expr: endNode(path) => vn - auto args = ArgumentList::make(pool); - args->addArgument(columnExpr); - auto endNode = FunctionCallExpression::make(pool, "endNode", args); - // expr: en[_dst] => dst vid - auto vidExpr = ConstantExpression::make(pool, kVid); - return AttributeExpression::make(pool, endNode, vidExpr); -} - -Expression* MatchSolver::getStartVidInPath(QueryContext* qctx, const std::string& colName) { - auto* pool = qctx->objPool(); - // expr: __Project_2[0] => path - auto columnExpr = InputPropertyExpression::make(pool, colName); - // expr: startNode(path) => v1 - auto args = ArgumentList::make(pool); - args->addArgument(columnExpr); - auto firstVertexExpr = FunctionCallExpression::make(pool, "startNode", args); - // expr: v1[_vid] => vid - return AttributeExpression::make(pool, firstVertexExpr, ConstantExpression::make(pool, kVid)); -} - PlanNode* MatchSolver::filtPathHasSameEdge(PlanNode* input, const std::string& column, QueryContext* qctx) { @@ -282,45 +226,6 @@ PlanNode* MatchSolver::filtPathHasSameEdge(PlanNode* input, return filter; } -Status MatchSolver::appendFetchVertexPlan(const Expression* nodeFilter, - const SpaceInfo& space, - QueryContext* qctx, - Expression** initialExpr, - SubPlan& plan) { - return appendFetchVertexPlan(nodeFilter, space, qctx, initialExpr, plan.root->outputVar(), plan); -} - -Status MatchSolver::appendFetchVertexPlan(const Expression* nodeFilter, - const SpaceInfo& space, - QueryContext* qctx, - Expression** initialExpr, - std::string inputVar, - SubPlan& plan) { - auto* pool = qctx->objPool(); - // [Project && Dedup] - extractAndDedupVidColumn(qctx, initialExpr, plan.root, inputVar, plan); - auto srcExpr = InputPropertyExpression::make(pool, kVid); - // [Get vertices] - auto props = SchemaUtil::getAllVertexProp(qctx, space.id, true); - NG_RETURN_IF_ERROR(props); - auto gv = GetVertices::make(qctx, plan.root, space.id, srcExpr, std::move(props).value(), {}); - - PlanNode* root = gv; - if (nodeFilter != nullptr) { - auto* newFilter = MatchSolver::rewriteLabel2Vertex(qctx, nodeFilter); - root = Filter::make(qctx, root, newFilter); - } - - // Normalize all columns to one - auto columns = pool->makeAndAdd(); - auto pathExpr = PathBuildExpression::make(pool); - pathExpr->add(VertexExpression::make(pool)); - columns->addColumn(new YieldColumn(pathExpr)); - plan.root = Project::make(qctx, root, columns); - plan.root->setColNames({kPathStr}); - return Status::OK(); -} - static YieldColumn* buildVertexColumn(ObjectPool* pool, const std::string& alias) { return new YieldColumn(InputPropertyExpression::make(pool, alias), alias); } diff --git a/src/graph/planner/match/MatchSolver.h b/src/graph/planner/match/MatchSolver.h index 4db5cf86b23..4a619e77c04 100644 --- a/src/graph/planner/match/MatchSolver.h +++ b/src/graph/planner/match/MatchSolver.h @@ -44,39 +44,10 @@ class MatchSolver final { QueryContext* qctx, bool isEdgeProperties = false); - static void extractAndDedupVidColumn(QueryContext* qctx, - Expression** initialExpr, - PlanNode* dep, - const std::string& inputVar, - SubPlan& plan); - - static Expression* initialExprOrEdgeDstExpr(QueryContext* qctx, - Expression** initialExpr, - const std::string& vidCol); - - static Expression* getEndVidInPath(QueryContext* qctx, const std::string& colName); - - static Expression* getStartVidInPath(QueryContext* qctx, const std::string& colName); - static PlanNode* filtPathHasSameEdge(PlanNode* input, const std::string& column, QueryContext* qctx); - static Status appendFetchVertexPlan(const Expression* nodeFilter, - const SpaceInfo& space, - QueryContext* qctx, - Expression** initialExpr, - SubPlan& plan); - - // In 0 step left expansion case, the result of initial index scan - // will be passed as inputVar after right expansion is finished - static Status appendFetchVertexPlan(const Expression* nodeFilter, - const SpaceInfo& space, - QueryContext* qctx, - Expression** initialExpr, - std::string inputVar, - SubPlan& plan); - // Build yield columns for match & shortestPath statement static void buildProjectColumns(QueryContext* qctx, Path& path, SubPlan& plan); }; diff --git a/src/graph/planner/ngql/PathPlanner.cpp b/src/graph/planner/ngql/PathPlanner.cpp index 8b20c266f3b..639c0257eec 100644 --- a/src/graph/planner/ngql/PathPlanner.cpp +++ b/src/graph/planner/ngql/PathPlanner.cpp @@ -322,11 +322,12 @@ PlanNode* PathPlanner::buildVertexPlan(PlanNode* dep, const std::string& input) // col 0 of the project->output is [node...] auto* unwindExpr = ColumnExpression::make(pool, 0); auto* unwind = Unwind::make(qctx, project, unwindExpr); + unwind->setFromPipe(true); unwind->setColNames({"nodes"}); // extract vid from vertex, col 0 is vertex auto idArgs = ArgumentList::make(pool); - idArgs->addArgument(ColumnExpression::make(pool, 1)); + idArgs->addArgument(ColumnExpression::make(pool, 0)); auto* src = FunctionCallExpression::make(pool, "id", idArgs); // get all vertexprop auto vertexProp = SchemaUtil::getAllVertexProp(qctx, pathCtx_->space.id, true); @@ -355,23 +356,24 @@ PlanNode* PathPlanner::buildEdgePlan(PlanNode* dep, const std::string& input) { // col 0 of the project->output() is [edge...] auto* unwindExpr = ColumnExpression::make(pool, 0); auto* unwind = Unwind::make(qctx, project, unwindExpr); + unwind->setFromPipe(true); unwind->setColNames({"edges"}); // extract src from edge auto srcArgs = ArgumentList::make(pool); - srcArgs->addArgument(ColumnExpression::make(pool, 1)); + srcArgs->addArgument(ColumnExpression::make(pool, 0)); auto* src = FunctionCallExpression::make(pool, "src", srcArgs); // extract dst from edge auto dstArgs = ArgumentList::make(pool); - dstArgs->addArgument(ColumnExpression::make(pool, 1)); + dstArgs->addArgument(ColumnExpression::make(pool, 0)); auto* dst = FunctionCallExpression::make(pool, "dst", dstArgs); // extract rank from edge auto rankArgs = ArgumentList::make(pool); - rankArgs->addArgument(ColumnExpression::make(pool, 1)); + rankArgs->addArgument(ColumnExpression::make(pool, 0)); auto* rank = FunctionCallExpression::make(pool, "rank", rankArgs); // type auto typeArgs = ArgumentList::make(pool); - typeArgs->addArgument(ColumnExpression::make(pool, 1)); + typeArgs->addArgument(ColumnExpression::make(pool, 0)); auto* type = FunctionCallExpression::make(pool, "typeid", typeArgs); // prepare edgetype auto edgeProp = SchemaUtil::getEdgeProps(qctx, pathCtx_->space, pathCtx_->over.edgeTypes, true); diff --git a/src/graph/planner/plan/Algo.cpp b/src/graph/planner/plan/Algo.cpp index 6007dadab21..1f42394f49a 100644 --- a/src/graph/planner/plan/Algo.cpp +++ b/src/graph/planner/plan/Algo.cpp @@ -6,6 +6,7 @@ #include "graph/planner/plan/Algo.h" #include "PlanNode.h" +#include "graph/planner/plan/PlanNodeVisitor.h" #include "graph/util/ToJson.h" namespace nebula { namespace graph { @@ -125,6 +126,10 @@ BiCartesianProduct::BiCartesianProduct(QueryContext* qctx, PlanNode* left, PlanN setColNames(lColNames); } +void BiCartesianProduct::accept(PlanNodeVisitor* visitor) { + visitor->visit(this); +} + BiCartesianProduct::BiCartesianProduct(QueryContext* qctx) : BinaryInputNode(qctx, Kind::kBiCartesianProduct) {} diff --git a/src/graph/planner/plan/Algo.h b/src/graph/planner/plan/Algo.h index 43b0caa2505..513e49259a1 100644 --- a/src/graph/planner/plan/Algo.h +++ b/src/graph/planner/plan/Algo.h @@ -313,6 +313,8 @@ class BiCartesianProduct final : public BinaryInputNode { PlanNode* clone() const override; + void accept(PlanNodeVisitor* visitor) override; + private: friend ObjectPool; diff --git a/src/graph/planner/plan/PlanNodeVisitor.h b/src/graph/planner/plan/PlanNodeVisitor.h index eb8b6d8d0da..91a29391239 100644 --- a/src/graph/planner/plan/PlanNodeVisitor.h +++ b/src/graph/planner/plan/PlanNodeVisitor.h @@ -6,6 +6,7 @@ #ifndef PLAN_PLANNODEVISITOR_H_ #define PLAN_PLANNODEVISITOR_H_ +#include "graph/planner/plan/Algo.h" #include "graph/planner/plan/PlanNode.h" #include "graph/planner/plan/Query.h" @@ -21,8 +22,12 @@ class PlanNodeVisitor { virtual void visit(Project *node) = 0; virtual void visit(Aggregate *node) = 0; virtual void visit(Traverse *node) = 0; + virtual void visit(ScanEdges *node) = 0; virtual void visit(AppendVertices *node) = 0; virtual void visit(BiJoin *node) = 0; + virtual void visit(Union *node) = 0; + virtual void visit(Unwind *node) = 0; + virtual void visit(BiCartesianProduct *node) = 0; }; } // namespace graph diff --git a/src/graph/planner/plan/Query.cpp b/src/graph/planner/plan/Query.cpp index aa1c8bed387..9f544fbc0b4 100644 --- a/src/graph/planner/plan/Query.cpp +++ b/src/graph/planner/plan/Query.cpp @@ -240,6 +240,10 @@ PlanNode* ScanEdges::clone() const { return newGE; } +void ScanEdges::accept(PlanNodeVisitor* visitor) { + visitor->visit(this); +} + void ScanEdges::cloneMembers(const ScanEdges& ge) { Explore::cloneMembers(ge); @@ -301,6 +305,10 @@ void Union::cloneMembers(const Union& f) { SetOp::cloneMembers(f); } +void Union::accept(PlanNodeVisitor* visitor) { + visitor->visit(this); +} + PlanNode* Intersect::clone() const { auto* newIntersect = Intersect::make(qctx_, nullptr, nullptr); newIntersect->cloneMembers(*this); @@ -369,6 +377,7 @@ std::unique_ptr Unwind::explain() const { PlanNode* Unwind::clone() const { auto* newUnwind = Unwind::make(qctx_, nullptr); + newUnwind->setFromPipe(fromPipe_); newUnwind->cloneMembers(*this); return newUnwind; } @@ -380,6 +389,10 @@ void Unwind::cloneMembers(const Unwind& p) { alias_ = p.alias(); } +void Unwind::accept(PlanNodeVisitor* visitor) { + visitor->visit(this); +} + std::unique_ptr Sort::explain() const { auto desc = SingleInputNode::explain(); addDescription("factors", folly::toJson(util::toJson(factorsString())), desc.get()); diff --git a/src/graph/planner/plan/Query.h b/src/graph/planner/plan/Query.h index 8144e27ef5d..bf4e2f564ab 100644 --- a/src/graph/planner/plan/Query.h +++ b/src/graph/planner/plan/Query.h @@ -670,6 +670,8 @@ class ScanEdges final : public Explore { PlanNode* clone() const override; std::unique_ptr explain() const override; + void accept(PlanNodeVisitor* visitor) override; + private: friend ObjectPool; ScanEdges(QueryContext* qctx, @@ -753,6 +755,8 @@ class Union final : public SetOp { return qctx->objPool()->makeAndAdd(qctx, left, right); } + void accept(PlanNodeVisitor* visitor) override; + PlanNode* clone() const override; private: @@ -837,13 +841,23 @@ class Unwind final : public SingleInputNode { return unwindExpr_; } - const std::string alias() const { + const std::string& alias() const { return alias_; } + bool fromPipe() const { + return fromPipe_; + } + + void setFromPipe(bool fromPipe) { + fromPipe_ = fromPipe; + } + PlanNode* clone() const override; std::unique_ptr explain() const override; + void accept(PlanNodeVisitor* visitor) override; + private: friend ObjectPool; Unwind(QueryContext* qctx, PlanNode* input, Expression* unwindExpr, std::string alias) @@ -854,6 +868,7 @@ class Unwind final : public SingleInputNode { private: Expression* unwindExpr_{nullptr}; std::string alias_; + bool fromPipe_{false}; }; // Sort the given record set. diff --git a/src/graph/service/GraphFlags.cpp b/src/graph/service/GraphFlags.cpp index 9029b393e94..be60913a10e 100644 --- a/src/graph/service/GraphFlags.cpp +++ b/src/graph/service/GraphFlags.cpp @@ -76,6 +76,8 @@ DEFINE_int32(max_sessions_per_ip_per_user, 300, "Maximum number of sessions that can be created per IP and per user"); +DEFINE_bool(optimize_appendvertices, false, "if true, return directly without go through RPC"); + // Sanity-checking Flag Values static bool ValidateSessIdleTimeout(const char* flagname, int32_t value) { // The max timeout is 604800 seconds(a week) diff --git a/src/graph/service/GraphFlags.h b/src/graph/service/GraphFlags.h index 480f97dff1a..f2c7ecf9c75 100644 --- a/src/graph/service/GraphFlags.h +++ b/src/graph/service/GraphFlags.h @@ -49,6 +49,7 @@ DECLARE_uint32(password_lock_time_in_secs); // Optimizer DECLARE_bool(enable_optimizer); +DECLARE_bool(optimize_appendvertice); DECLARE_int64(max_allowed_connections); diff --git a/src/graph/service/PermissionCheck.cpp b/src/graph/service/PermissionCheck.cpp index 8939a249ae5..da0dd3789dd 100644 --- a/src/graph/service/PermissionCheck.cpp +++ b/src/graph/service/PermissionCheck.cpp @@ -131,6 +131,7 @@ namespace graph { case Sentence::Kind::kGetSubgraph: case Sentence::Kind::kLimit: case Sentence::Kind::kGroupBy: + case Sentence::Kind::kUnwind: case Sentence::Kind::kReturn: { return PermissionManager::canReadSchemaOrData(session, vctx); } diff --git a/src/graph/validator/CMakeLists.txt b/src/graph/validator/CMakeLists.txt index ac7dd18981e..9c8c7f66fbf 100644 --- a/src/graph/validator/CMakeLists.txt +++ b/src/graph/validator/CMakeLists.txt @@ -27,6 +27,7 @@ nebula_add_library( FindPathValidator.cpp LookupValidator.cpp MatchValidator.cpp + UnwindValidator.cpp ) nebula_add_subdirectory(test) diff --git a/src/graph/validator/FetchEdgesValidator.cpp b/src/graph/validator/FetchEdgesValidator.cpp index bcc45c9f6fd..57b9477e82d 100644 --- a/src/graph/validator/FetchEdgesValidator.cpp +++ b/src/graph/validator/FetchEdgesValidator.cpp @@ -42,8 +42,7 @@ Status FetchEdgesValidator::validateEdgeName() { // Check validity of edge key(src, type, rank, dst) // from Input/Variable expression specified in sentence -StatusOr FetchEdgesValidator::validateEdgeRef(const Expression *expr, - Value::Type type) { +StatusOr FetchEdgesValidator::validateEdgeRef(const Expression *expr) { const auto &kind = expr->kind(); if (kind != Expression::Kind::kInputProperty && kind != EdgeExpression::Kind::kVarProperty) { return Status::SemanticError("`%s', only input and variable expression is acceptable", @@ -51,12 +50,6 @@ StatusOr FetchEdgesValidator::validateEdgeRef(const Expression *exp } auto exprType = deduceExprType(expr); NG_RETURN_IF_ERROR(exprType); - if (exprType.value() != type) { - std::stringstream ss; - ss << "`" << expr->toString() << "' should be type of " << type << ", but was " - << exprType.value(); - return Status::SemanticError(ss.str()); - } if (kind == Expression::Kind::kInputProperty) { return inputVarName_; } @@ -72,13 +65,22 @@ Status FetchEdgesValidator::validateEdgeKey() { std::string inputVarName; if (sentence->isRef()) { // edge keys from Input/Variable auto *srcExpr = sentence->ref()->srcid(); - auto result = validateEdgeRef(srcExpr, vidType_); + auto result = validateEdgeRef(srcExpr); NG_RETURN_IF_ERROR(result); inputVarName = std::move(result).value(); auto *rankExpr = sentence->ref()->rank(); if (rankExpr->kind() != Expression::Kind::kConstant) { - result = validateEdgeRef(rankExpr, Value::Type::INT); + auto rankType = deduceExprType(rankExpr); + NG_RETURN_IF_ERROR(rankType); + if (rankType.value() != Value::Type::INT) { + std::stringstream ss; + ss << "`" << rankExpr->toString() << "' should be type of INT, but was " + << rankType.value(); + return Status::SemanticError(ss.str()); + } + + result = validateEdgeRef(rankExpr); NG_RETURN_IF_ERROR(result); if (inputVarName != result.value()) { return Status::SemanticError( @@ -88,7 +90,7 @@ Status FetchEdgesValidator::validateEdgeKey() { } auto *dstExpr = sentence->ref()->dstid(); - result = validateEdgeRef(dstExpr, vidType_); + result = validateEdgeRef(dstExpr); NG_RETURN_IF_ERROR(result); if (inputVarName != result.value()) { return Status::SemanticError( diff --git a/src/graph/validator/FetchEdgesValidator.h b/src/graph/validator/FetchEdgesValidator.h index 0d1ce562640..2f652af5196 100644 --- a/src/graph/validator/FetchEdgesValidator.h +++ b/src/graph/validator/FetchEdgesValidator.h @@ -22,7 +22,7 @@ class FetchEdgesValidator final : public Validator { Status validateEdgeName(); - StatusOr validateEdgeRef(const Expression* expr, Value::Type type); + StatusOr validateEdgeRef(const Expression* expr); Status validateEdgeKey(); diff --git a/src/graph/validator/UnwindValidator.cpp b/src/graph/validator/UnwindValidator.cpp new file mode 100644 index 00000000000..6cee4c8886d --- /dev/null +++ b/src/graph/validator/UnwindValidator.cpp @@ -0,0 +1,50 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#include "graph/validator/UnwindValidator.h" + +#include "graph/context/QueryContext.h" +#include "graph/planner/plan/Query.h" +#include "graph/util/ExpressionUtils.h" +#include "parser/TraverseSentences.h" + +namespace nebula { +namespace graph { + +Status UnwindValidator::validateImpl() { + auto *unwindSentence = static_cast(sentence_); + if (unwindSentence->alias().empty()) { + return Status::SemanticError("Expression in UNWIND must be aliased (use AS)"); + } + alias_ = unwindSentence->alias(); + unwindExpr_ = unwindSentence->expr()->clone(); + + if (ExpressionUtils::findAny(unwindExpr_, + {Expression::Kind::kSrcProperty, + Expression::Kind::kDstProperty, + Expression::Kind::kVarProperty, + Expression::Kind::kLabel, + Expression::Kind::kAggregate, + Expression::Kind::kEdgeProperty})) { + return Status::SemanticError("Not support `%s' in UNWIND sentence.", + unwindExpr_->toString().c_str()); + } + + auto typeStatus = deduceExprType(unwindExpr_); + NG_RETURN_IF_ERROR(typeStatus); + outputs_.emplace_back(alias_, Value::Type::__EMPTY__); + return Status::OK(); +} + +Status UnwindValidator::toPlan() { + auto *unwind = Unwind::make(qctx_, nullptr, unwindExpr_, alias_); + unwind->setColNames({alias_}); + unwind->setFromPipe(true); + root_ = tail_ = unwind; + return Status::OK(); +} + +} // namespace graph +} // namespace nebula diff --git a/src/graph/validator/UnwindValidator.h b/src/graph/validator/UnwindValidator.h new file mode 100644 index 00000000000..66aa3aaaa79 --- /dev/null +++ b/src/graph/validator/UnwindValidator.h @@ -0,0 +1,36 @@ +/* Copyright (c) 2020 vesoft inc. All rights reserved. + * + * This source code is licensed under Apache 2.0 License. + */ + +#ifndef GRAPH_VALIDATOR_UNWINDVALIDATOR_H_ +#define GRAPH_VALIDATOR_UNWINDVALIDATOR_H_ + +#include "graph/planner/plan/Query.h" +#include "graph/validator/Validator.h" + +namespace nebula { + +class ObjectPool; + +namespace graph { + +class QueryContext; + +class UnwindValidator final : public Validator { + public: + UnwindValidator(Sentence *sentence, QueryContext *context) : Validator(sentence, context) {} + + Status validateImpl() override; + + Status toPlan() override; + + private: + Expression *unwindExpr_{nullptr}; + std::string alias_; +}; + +} // namespace graph +} // namespace nebula + +#endif // GRAPH_VALIDATOR_UNWINDVALIDATOR_H_ diff --git a/src/graph/validator/Validator.cpp b/src/graph/validator/Validator.cpp index 58330d75132..a98dde58ad6 100644 --- a/src/graph/validator/Validator.cpp +++ b/src/graph/validator/Validator.cpp @@ -33,6 +33,7 @@ #include "graph/validator/ReportError.h" #include "graph/validator/SequentialValidator.h" #include "graph/validator/SetValidator.h" +#include "graph/validator/UnwindValidator.h" #include "graph/validator/UseValidator.h" #include "graph/validator/YieldValidator.h" #include "graph/visitor/DeduceTypeVisitor.h" @@ -252,6 +253,8 @@ std::unique_ptr Validator::makeValidator(Sentence* sentence, QueryCon return std::make_unique(sentence, context); case Sentence::Kind::kClearSpace: return std::make_unique(sentence, context); + case Sentence::Kind::kUnwind: + return std::make_unique(sentence, context); case Sentence::Kind::kUnknown: case Sentence::Kind::kReturn: { // nothing @@ -449,15 +452,16 @@ Status Validator::validateStarts(const VerticesClause* clause, Starts& starts) { src->toString().c_str()); } starts.fromType = src->kind() == Expression::Kind::kInputProperty ? kPipe : kVariable; - auto type = deduceExprType(src); - if (!type.ok()) { - return type.status(); + auto res = deduceExprType(src); + if (!res.ok()) { + return res.status(); } auto vidType = space_.spaceDesc.vid_type_ref().value().get_type(); - if (type.value() != SchemaUtil::propTypeToValueType(vidType)) { + auto& type = res.value(); + if (type != Value::Type::__EMPTY__ && type != SchemaUtil::propTypeToValueType(vidType)) { std::stringstream ss; ss << "`" << src->toString() << "', the srcs should be type of " - << apache::thrift::util::enumNameSafe(vidType) << ", but was`" << type.value() << "'"; + << apache::thrift::util::enumNameSafe(vidType) << ", but was`" << type << "'"; return Status::SemanticError(ss.str()); } starts.originalSrc = src; diff --git a/src/graph/validator/test/FetchEdgesTest.cpp b/src/graph/validator/test/FetchEdgesTest.cpp index 4874bbc4f02..190453dd714 100644 --- a/src/graph/validator/test/FetchEdgesTest.cpp +++ b/src/graph/validator/test/FetchEdgesTest.cpp @@ -190,13 +190,7 @@ TEST_F(FetchEdgesValidatorTest, FetchEdgesProp) { auto *project = Project::make(qctx, filter, yieldColumns.get()); auto *dedup = Dedup::make(qctx, project); - // data collect - auto *dataCollect = DataCollect::make(qctx, DataCollect::DCKind::kRowBasedMove); - dataCollect->addDep(dedup); - dataCollect->setInputVars({dedup->outputVar()}); - dataCollect->setColNames({"like.start", "like.end"}); - - auto result = Eq(qctx->plan()->root(), dataCollect); + auto result = Eq(qctx->plan()->root(), dedup); ASSERT_TRUE(result.ok()) << result; } } diff --git a/src/graph/validator/test/FetchVerticesTest.cpp b/src/graph/validator/test/FetchVerticesTest.cpp index a7e62c4b24d..0f6d676b54e 100644 --- a/src/graph/validator/test/FetchVerticesTest.cpp +++ b/src/graph/validator/test/FetchVerticesTest.cpp @@ -364,13 +364,7 @@ TEST_F(FetchVerticesValidatorTest, FetchVerticesProp) { auto *dedup = Dedup::make(qctx, project); - // data collect - auto *dataCollect = DataCollect::make(qctx, DataCollect::DCKind::kRowBasedMove); - dataCollect->addDep(dedup); - dataCollect->setInputVars({dedup->outputVar()}); - dataCollect->setColNames({"person.name", "person.age"}); - - auto result = Eq(qctx->plan()->root(), dataCollect); + auto result = Eq(qctx->plan()->root(), dedup); ASSERT_TRUE(result.ok()) << result; } // ON * diff --git a/src/graph/validator/test/GetSubgraphValidatorTest.cpp b/src/graph/validator/test/GetSubgraphValidatorTest.cpp index 8eae1c3b757..57b4ea6324a 100644 --- a/src/graph/validator/test/GetSubgraphValidatorTest.cpp +++ b/src/graph/validator/test/GetSubgraphValidatorTest.cpp @@ -218,8 +218,7 @@ TEST_F(GetSubgraphValidatorTest, RefNotExist) { "PROP FROM $-.id YIELD vertices as nodes"; auto result = checkResult(query); EXPECT_EQ(std::string(result.message()), - "SemanticError: `$-.id', the srcs should be type of " - "FIXED_STRING, but was`INT'"); + "SemanticError: `$-.id', the srcs should be type of FIXED_STRING, but was`INT'"); } { std::string query = @@ -227,8 +226,7 @@ TEST_F(GetSubgraphValidatorTest, RefNotExist) { "FROM $a.ID YIELD edges as relationships"; auto result = checkResult(query); EXPECT_EQ(std::string(result.message()), - "SemanticError: `$a.ID', the srcs should be type of " - "FIXED_STRING, but was`INT'"); + "SemanticError: `$a.ID', the srcs should be type of FIXED_STRING, but was`INT'"); } { std::string query = diff --git a/src/graph/validator/test/MatchValidatorTest.cpp b/src/graph/validator/test/MatchValidatorTest.cpp index 2356e2da7b4..e81157f7c00 100644 --- a/src/graph/validator/test/MatchValidatorTest.cpp +++ b/src/graph/validator/test/MatchValidatorTest.cpp @@ -118,8 +118,7 @@ TEST_F(MatchValidatorTest, groupby) { "avg(distinct n.person.age)+1 AS age," "labels(n) AS lb " "ORDER BY id;"; - std::vector expected = {PlanNode::Kind::kDataCollect, - PlanNode::Kind::kSort, + std::vector expected = {PlanNode::Kind::kSort, PlanNode::Kind::kProject, PlanNode::Kind::kAggregate, PlanNode::Kind::kProject, @@ -140,8 +139,7 @@ TEST_F(MatchValidatorTest, groupby) { "labels(n) AS lb " "ORDER BY id " "SKIP 10 LIMIT 20;"; - std::vector expected = {PlanNode::Kind::kDataCollect, - PlanNode::Kind::kLimit, + std::vector expected = {PlanNode::Kind::kLimit, PlanNode::Kind::kSort, PlanNode::Kind::kProject, PlanNode::Kind::kAggregate, @@ -220,8 +218,7 @@ TEST_F(MatchValidatorTest, groupby) { "avg(distinct n.person.age) AS age," "labels(m) AS lb " "SKIP 10 LIMIT 20;"; - std::vector expected = {PlanNode::Kind::kDataCollect, - PlanNode::Kind::kLimit, + std::vector expected = {PlanNode::Kind::kLimit, PlanNode::Kind::kAggregate, PlanNode::Kind::kFilter, PlanNode::Kind::kProject, @@ -242,8 +239,7 @@ TEST_F(MatchValidatorTest, groupby) { "min(n.person.age) AS min," "avg(distinct n.person.age) AS age," "labels(m) AS lb;"; - std::vector expected = {PlanNode::Kind::kDataCollect, - PlanNode::Kind::kDedup, + std::vector expected = {PlanNode::Kind::kDedup, PlanNode::Kind::kAggregate, PlanNode::Kind::kFilter, PlanNode::Kind::kProject, @@ -265,8 +261,7 @@ TEST_F(MatchValidatorTest, groupby) { "avg(distinct n.person.age)+1 AS age," "labels(m) AS lb " "SKIP 10 LIMIT 20;"; - std::vector expected = {PlanNode::Kind::kDataCollect, - PlanNode::Kind::kLimit, + std::vector expected = {PlanNode::Kind::kLimit, PlanNode::Kind::kProject, PlanNode::Kind::kAggregate, PlanNode::Kind::kFilter, @@ -290,8 +285,7 @@ TEST_F(MatchValidatorTest, groupby) { "labels(m) AS lb " "ORDER BY id " "SKIP 10 LIMIT 20;"; - std::vector expected = {PlanNode::Kind::kDataCollect, - PlanNode::Kind::kLimit, + std::vector expected = {PlanNode::Kind::kLimit, PlanNode::Kind::kSort, PlanNode::Kind::kDedup, PlanNode::Kind::kProject, @@ -317,8 +311,7 @@ TEST_F(MatchValidatorTest, groupby) { "labels(m) AS lb " "ORDER BY id " "SKIP 10 LIMIT 20;"; - std::vector expected = {PlanNode::Kind::kDataCollect, - PlanNode::Kind::kLimit, + std::vector expected = {PlanNode::Kind::kLimit, PlanNode::Kind::kSort, PlanNode::Kind::kDedup, PlanNode::Kind::kAggregate, @@ -343,8 +336,7 @@ TEST_F(MatchValidatorTest, groupby) { "labels(m) AS lb " "ORDER BY id " "SKIP 10 LIMIT 20;"; - std::vector expected = {PlanNode::Kind::kDataCollect, - PlanNode::Kind::kLimit, + std::vector expected = {PlanNode::Kind::kLimit, PlanNode::Kind::kSort, PlanNode::Kind::kDedup, PlanNode::Kind::kProject, diff --git a/src/graph/validator/test/QueryValidatorTest.cpp b/src/graph/validator/test/QueryValidatorTest.cpp index c1a1e62275e..49ea0c03d16 100644 --- a/src/graph/validator/test/QueryValidatorTest.cpp +++ b/src/graph/validator/test/QueryValidatorTest.cpp @@ -107,7 +107,6 @@ TEST_F(QueryValidatorTest, GoNSteps) { "GO 3 steps FROM \"1\",\"2\",\"3\" OVER like WHERE $^.person.age > 20" "YIELD distinct $^.person.name"; std::vector expected = { - PK::kDataCollect, PK::kDedup, PK::kProject, PK::kFilter, @@ -125,8 +124,7 @@ TEST_F(QueryValidatorTest, GoNSteps) { std::string query = "GO 2 STEPS FROM \"1\",\"2\",\"3\" OVER like WHERE $^.person.age > 20" "YIELD distinct $^.person.name "; - std::vector expected = {PK::kDataCollect, - PK::kDedup, + std::vector expected = {PK::kDedup, PK::kProject, PK::kFilter, PK::kGetNeighbors, @@ -239,8 +237,7 @@ TEST_F(QueryValidatorTest, GoWithPipe) { "GO 1 STEPS FROM \"1\" OVER like YIELD like._dst AS " "id | GO 1 STEPS FROM $-.id OVER like " "WHERE $-.id == \"2\" YIELD DISTINCT $-.id, like._dst"; - std::vector expected = {PK::kDataCollect, - PK::kDedup, + std::vector expected = {PK::kDedup, PK::kProject, PK::kFilter, PK::kInnerJoin, @@ -286,11 +283,11 @@ TEST_F(QueryValidatorTest, GoWithPipe) { "id | GO 2 STEPS FROM $-.id OVER like " "WHERE $-.id == \"2\" YIELD DISTINCT $-.id, like._dst"; std::vector expected = { - PK::kDataCollect, PK::kDedup, PK::kProject, PK::kFilter, PK::kInnerJoin, - PK::kInnerJoin, PK::kProject, PK::kGetNeighbors, PK::kLoop, PK::kDedup, - PK::kDedup, PK::kProject, PK::kProject, PK::kDedup, PK::kInnerJoin, - PK::kProject, PK::kDedup, PK::kProject, PK::kProject, PK::kGetNeighbors, - PK::kDedup, PK::kStart, PK::kProject, PK::kGetNeighbors, PK::kStart, + PK::kDedup, PK::kProject, PK::kFilter, PK::kInnerJoin, PK::kInnerJoin, + PK::kProject, PK::kGetNeighbors, PK::kLoop, PK::kDedup, PK::kDedup, + PK::kProject, PK::kProject, PK::kDedup, PK::kInnerJoin, PK::kProject, + PK::kDedup, PK::kProject, PK::kProject, PK::kGetNeighbors, PK::kDedup, + PK::kStart, PK::kProject, PK::kGetNeighbors, PK::kStart, }; EXPECT_TRUE(checkResult(query, expected)); } @@ -327,7 +324,6 @@ TEST_F(QueryValidatorTest, GoWithPipe) { "YIELD DISTINCT $-.name, like.likeness + 1, $-.id, like._dst, " "$$.person.name"; std::vector expected = { - PK::kDataCollect, PK::kDedup, PK::kProject, PK::kInnerJoin, @@ -393,13 +389,12 @@ TEST_F(QueryValidatorTest, GoWithPipe) { "YIELD DISTINCT $-.name, like.likeness + 1, $-.id, like._dst, " "$$.person.name"; std::vector expected = { - PK::kDataCollect, PK::kDedup, PK::kProject, PK::kInnerJoin, PK::kInnerJoin, - PK::kLeftJoin, PK::kProject, PK::kGetVertices, PK::kProject, PK::kGetNeighbors, - PK::kLoop, PK::kDedup, PK::kDedup, PK::kProject, PK::kProject, - PK::kDedup, PK::kInnerJoin, PK::kProject, PK::kDedup, PK::kProject, - PK::kProject, PK::kLeftJoin, PK::kDedup, PK::kProject, PK::kProject, - PK::kGetVertices, PK::kGetNeighbors, PK::kProject, PK::kStart, PK::kGetNeighbors, - PK::kStart, + PK::kDedup, PK::kProject, PK::kInnerJoin, PK::kInnerJoin, PK::kLeftJoin, + PK::kProject, PK::kGetVertices, PK::kProject, PK::kGetNeighbors, PK::kLoop, + PK::kDedup, PK::kDedup, PK::kProject, PK::kProject, PK::kDedup, + PK::kInnerJoin, PK::kProject, PK::kDedup, PK::kProject, PK::kProject, + PK::kLeftJoin, PK::kDedup, PK::kProject, PK::kProject, PK::kGetVertices, + PK::kGetNeighbors, PK::kProject, PK::kStart, PK::kGetNeighbors, PK::kStart, }; EXPECT_TRUE(checkResult(query, expected)); } @@ -603,7 +598,6 @@ TEST_F(QueryValidatorTest, GoOneStep) { "YIELD DISTINCT $^.person.name, like._dst, " "$$.person.name, $$.person.age + 1"; std::vector expected = { - PK::kDataCollect, PK::kDedup, PK::kProject, PK::kFilter, @@ -641,7 +635,6 @@ TEST_F(QueryValidatorTest, GoOneStep) { "GO FROM \"1\",\"2\",\"3\" OVER like WHERE $^.person.age > 20" "YIELD distinct $^.person.name "; std::vector expected = { - PK::kDataCollect, PK::kDedup, PK::kProject, PK::kFilter, @@ -1011,7 +1004,7 @@ TEST_F(QueryValidatorTest, Limit) { { std::string query = "GO FROM \"Ann\" OVER like YIELD like._dst AS like | LIMIT 1, 3"; std::vector expected = { - PK::kDataCollect, PK::kLimit, PK::kProject, PK::kGetNeighbors, PK::kStart}; + PK::kLimit, PK::kProject, PK::kGetNeighbors, PK::kStart}; EXPECT_TRUE(checkResult(query, expected)); } } @@ -1021,8 +1014,7 @@ TEST_F(QueryValidatorTest, OrderBy) { std::string query = "GO FROM \"Ann\" OVER like YIELD $^.person.age AS age" " | ORDER BY $-.age"; - std::vector expected = { - PK::kDataCollect, PK::kSort, PK::kProject, PK::kGetNeighbors, PK::kStart}; + std::vector expected = {PK::kSort, PK::kProject, PK::kGetNeighbors, PK::kStart}; EXPECT_TRUE(checkResult(query, expected)); } // not exist factor @@ -1040,7 +1032,7 @@ TEST_F(QueryValidatorTest, OrderByAndLimit) { "GO FROM \"Ann\" OVER like YIELD $^.person.age AS age" " | ORDER BY $-.age | LIMIT 1"; std::vector expected = { - PK::kDataCollect, PK::kLimit, PK::kSort, PK::kProject, PK::kGetNeighbors, PK::kStart}; + PK::kLimit, PK::kSort, PK::kProject, PK::kGetNeighbors, PK::kStart}; EXPECT_TRUE(checkResult(query, expected)); } } @@ -1053,7 +1045,6 @@ TEST_F(QueryValidatorTest, TestSetValidator) { "\"2\" " "OVER like YIELD like.start AS start"; std::vector expected = { - PK::kDataCollect, PK::kUnion, PK::kProject, PK::kProject, @@ -1072,7 +1063,6 @@ TEST_F(QueryValidatorTest, TestSetValidator) { "YIELD " "like.start AS start"; std::vector expected = { - PK::kDataCollect, PK::kDedup, PK::kUnion, PK::kDedup, @@ -1096,7 +1086,6 @@ TEST_F(QueryValidatorTest, TestSetValidator) { "FROM \"2\" " "OVER like YIELD like.start AS start"; std::vector expected = { - PK::kDataCollect, PK::kDedup, PK::kUnion, PK::kProject, @@ -1121,7 +1110,6 @@ TEST_F(QueryValidatorTest, TestSetValidator) { "GO FROM \"1\" OVER like YIELD like.start AS start INTERSECT GO FROM " "\"2\" OVER like YIELD like.start AS start"; std::vector expected = { - PK::kDataCollect, PK::kIntersect, PK::kProject, PK::kProject, @@ -1138,7 +1126,6 @@ TEST_F(QueryValidatorTest, TestSetValidator) { "GO FROM \"1\" OVER like YIELD like.start AS start MINUS GO FROM " "\"2\" OVER like YIELD like.start AS start"; std::vector expected = { - PK::kDataCollect, PK::kMinus, PK::kProject, PK::kProject, @@ -1209,7 +1196,6 @@ TEST_F(QueryValidatorTest, TestMatch) { "MATCH (:person{name:'Dwyane Wade'}) -[:like]-> () -[:like]-> (v3) " "RETURN DISTINCT v3.person.name AS Name"; std::vector expected = { - PK::kDataCollect, PK::kDedup, PK::kProject, PK::kProject, diff --git a/src/graph/validator/test/YieldValidatorTest.cpp b/src/graph/validator/test/YieldValidatorTest.cpp index b436933ff14..fa4d47ef13e 100644 --- a/src/graph/validator/test/YieldValidatorTest.cpp +++ b/src/graph/validator/test/YieldValidatorTest.cpp @@ -523,7 +523,6 @@ TEST_F(YieldValidatorTest, AggCall) { "like.likeness AS like" "| YIELD DISTINCT AVG($-.age + 1), SUM($-.like), COUNT(*)"; expected_ = { - PlanNode::Kind::kDataCollect, PlanNode::Kind::kDedup, PlanNode::Kind::kAggregate, PlanNode::Kind::kProject, diff --git a/src/graph/visitor/FoldConstantExprVisitor.cpp b/src/graph/visitor/FoldConstantExprVisitor.cpp index 30cadcf8b25..16633288665 100644 --- a/src/graph/visitor/FoldConstantExprVisitor.cpp +++ b/src/graph/visitor/FoldConstantExprVisitor.cpp @@ -349,7 +349,7 @@ void FoldConstantExprVisitor::visitBinaryExpr(BinaryExpression *expr) { Expression *FoldConstantExprVisitor::fold(Expression *expr) { // Container expression should remain the same type after being folded - if (expr->isContainerExpr()) { + if (expr->isContainerExpr() || !status_.ok()) { return expr; } diff --git a/src/graph/visitor/PropertyTrackerVisitor.cpp b/src/graph/visitor/PropertyTrackerVisitor.cpp index f12871829b9..5ec7e8629f5 100644 --- a/src/graph/visitor/PropertyTrackerVisitor.cpp +++ b/src/graph/visitor/PropertyTrackerVisitor.cpp @@ -5,14 +5,56 @@ #include "graph/visitor/PropertyTrackerVisitor.h" -#include -#include - #include "common/expression/Expression.h" #include "graph/context/QueryContext.h" namespace nebula { namespace graph { +void PropertyTracker::insertVertexProp(const std::string &name, + TagID tagId, + const std::string &propName) { + if (colsSet.find(name) != colsSet.end()) { + return; + } + auto iter = vertexPropsMap.find(name); + if (iter == vertexPropsMap.end()) { + vertexPropsMap[name][tagId].emplace(propName); + } else { + auto propIter = iter->second.find(tagId); + if (propIter == iter->second.end()) { + std::unordered_set temp({propName}); + iter->second.emplace(tagId, std::move(temp)); + } else { + propIter->second.emplace(propName); + } + } +} + +void PropertyTracker::insertEdgeProp(const std::string &name, + EdgeType type, + const std::string &propName) { + if (colsSet.find(name) != colsSet.end()) { + return; + } + auto iter = edgePropsMap.find(name); + if (iter == edgePropsMap.end()) { + edgePropsMap[name][type].emplace(propName); + } else { + auto propIter = iter->second.find(type); + if (propIter == iter->second.end()) { + std::unordered_set temp({propName}); + iter->second.emplace(type, std::move(temp)); + } else { + propIter->second.emplace(propName); + } + } +} + +void PropertyTracker::insertCols(const std::string &name) { + colsSet.emplace(name); + vertexPropsMap.erase(name); + edgePropsMap.erase(name); +} Status PropertyTracker::update(const std::string &oldName, const std::string &newName) { if (oldName == newName) { @@ -32,6 +74,7 @@ Status PropertyTracker::update(const std::string &oldName, const std::string &ne } vertexPropsMap[newName] = std::move(it1->second); vertexPropsMap.erase(it1); + colsSet.erase(oldName); } if (hasEdgeAlias) { if (edgePropsMap.find(newName) != edgePropsMap.end()) { @@ -39,12 +82,13 @@ Status PropertyTracker::update(const std::string &oldName, const std::string &ne } edgePropsMap[newName] = std::move(it2->second); edgePropsMap.erase(it2); + colsSet.erase(oldName); } auto it3 = colsSet.find(oldName); if (it3 != colsSet.end()) { colsSet.erase(it3); - colsSet.insert(newName); + insertCols(newName); } return Status::OK(); @@ -72,7 +116,7 @@ void PropertyTrackerVisitor::visit(TagPropertyExpression *expr) { return; } auto tagId = ret.value(); - propsUsed_.vertexPropsMap[entityAlias_][tagId].emplace(propName); + propsUsed_.insertVertexProp(entityAlias_, tagId, propName); } void PropertyTrackerVisitor::visit(EdgePropertyExpression *expr) { @@ -84,7 +128,7 @@ void PropertyTrackerVisitor::visit(EdgePropertyExpression *expr) { return; } auto edgeType = ret.value(); - propsUsed_.edgePropsMap[entityAlias_][edgeType].emplace(propName); + propsUsed_.insertEdgeProp(entityAlias_, edgeType, propName); } void PropertyTrackerVisitor::visit(LabelTagPropertyExpression *expr) { @@ -102,17 +146,17 @@ void PropertyTrackerVisitor::visit(LabelTagPropertyExpression *expr) { return; } auto tagId = ret.value(); - propsUsed_.vertexPropsMap[nodeAlias][tagId].emplace(propName); + propsUsed_.insertVertexProp(nodeAlias, tagId, propName); } void PropertyTrackerVisitor::visit(InputPropertyExpression *expr) { auto &colName = expr->prop(); - propsUsed_.colsSet.emplace(colName); + propsUsed_.insertCols(colName); } void PropertyTrackerVisitor::visit(VariablePropertyExpression *expr) { auto &colName = expr->prop(); - propsUsed_.colsSet.emplace(colName); + propsUsed_.insertCols(colName); } void PropertyTrackerVisitor::visit(AttributeExpression *expr) { @@ -127,28 +171,79 @@ void PropertyTrackerVisitor::visit(AttributeExpression *expr) { return; } auto &propName = constVal.getStr(); - static const int kUnknownEdgeType = 0; switch (lhs->kind()) { + case Expression::Kind::kInputProperty: case Expression::Kind::kVarProperty: { // $e.name - auto *varPropExpr = static_cast(lhs); + auto *varPropExpr = static_cast(lhs); auto &edgeAlias = varPropExpr->prop(); - propsUsed_.edgePropsMap[edgeAlias][kUnknownEdgeType].emplace(propName); - break; - } - case Expression::Kind::kInputProperty: { - auto *inputPropExpr = static_cast(lhs); - auto &edgeAlias = inputPropExpr->prop(); - propsUsed_.edgePropsMap[edgeAlias][kUnknownEdgeType].emplace(propName); + propsUsed_.insertEdgeProp(edgeAlias, unKnowType_, propName); break; } case Expression::Kind::kSubscript: { // $-.e[0].name auto *subscriptExpr = static_cast(lhs); auto *subLeftExpr = subscriptExpr->left(); - if (subLeftExpr->kind() == Expression::Kind::kInputProperty) { - auto *inputPropExpr = static_cast(subLeftExpr); - auto &edgeAlias = inputPropExpr->prop(); - propsUsed_.edgePropsMap[edgeAlias][kUnknownEdgeType].emplace(propName); + auto kind = subLeftExpr->kind(); + if (kind == Expression::Kind::kInputProperty || kind == Expression::Kind::kVarProperty) { + auto *propExpr = static_cast(subLeftExpr); + auto &edgeAlias = propExpr->prop(); + propsUsed_.insertEdgeProp(edgeAlias, unKnowType_, propName); + } else if (kind == Expression::Kind::kListComprehension) { + // match (src_v:player{name:"Manu Ginobili"})-[e*2]-(dst_v) return e[0].start_year + auto *listExpr = static_cast(subLeftExpr); + auto *collectExpr = listExpr->collection(); + if (collectExpr->kind() == Expression::Kind::kInputProperty) { + auto *inputPropExpr = static_cast(collectExpr); + auto &aliasName = inputPropExpr->prop(); + propsUsed_.insertEdgeProp(aliasName, unKnowType_, propName); + } + } + break; + } + case Expression::Kind::kFunctionCall: { // properties(t3).name + auto *funCallExpr = static_cast(lhs); + auto funName = funCallExpr->name(); + std::transform(funName.begin(), funName.end(), funName.begin(), ::tolower); + if (funName != "properties") { + break; + } + auto *argExpr = funCallExpr->args()->args().front(); + auto kind = argExpr->kind(); + switch (kind) { + case Expression::Kind::kVarProperty: + case Expression::Kind::kInputProperty: { + // match (v) return properties(v).name + auto *inputPropExpr = static_cast(argExpr); + auto &aliasName = inputPropExpr->prop(); + propsUsed_.insertVertexProp(aliasName, unKnowType_, propName); + break; + } + case Expression::Kind::kSubscript: { + auto *subscriptExpr = static_cast(argExpr); + auto *subLeftExpr = subscriptExpr->left(); + auto leftKind = subLeftExpr->kind(); + if (leftKind == Expression::Kind::kInputProperty || + leftKind == Expression::Kind::kVarProperty) { + // match (v)-[e]->(b) return properties(e).start_year + auto *propExpr = static_cast(subLeftExpr); + auto &aliasName = propExpr->prop(); + propsUsed_.insertEdgeProp(aliasName, unKnowType_, propName); + } else if (leftKind == Expression::Kind::kListComprehension) { + // match (v)-[c*2]->(b) retrun properties(c[0]).start_year + // properties([e IN $-.c WHERE is_edge($e)][0]).start_year + auto *listExpr = static_cast(subLeftExpr); + auto *collectExpr = listExpr->collection(); + if (collectExpr->kind() == Expression::Kind::kInputProperty) { + auto *inputPropExpr = static_cast(collectExpr); + auto &aliasName = inputPropExpr->prop(); + propsUsed_.insertEdgeProp(aliasName, unKnowType_, propName); + } + } + break; + } + default: + break; } + break; } default: break; @@ -156,32 +251,12 @@ void PropertyTrackerVisitor::visit(AttributeExpression *expr) { } void PropertyTrackerVisitor::visit(FunctionCallExpression *expr) { - static const std::unordered_set kVertexIgnoreFuncs = {"id"}; - static const std::unordered_set kEdgeIgnoreFuncs = { - "src", "dst", "type", "typeid", "rank"}; + static const std::unordered_set ignoreFuncs = { + "src", "dst", "type", "typeid", "id", "rank", "length"}; auto funName = expr->name(); - if (kVertexIgnoreFuncs.find(funName) != kVertexIgnoreFuncs.end()) { - DCHECK_EQ(expr->args()->numArgs(), 1); - auto argExpr = expr->args()->args()[0]; - auto nodeAlias = extractColNameFromInputPropOrVarPropExpr(argExpr); - if (!nodeAlias.empty()) { - auto it = propsUsed_.vertexPropsMap.find(nodeAlias); - if (it == propsUsed_.vertexPropsMap.end()) { - propsUsed_.vertexPropsMap[nodeAlias] = {}; - } - } - return; - } else if (kEdgeIgnoreFuncs.find(funName) != kEdgeIgnoreFuncs.end()) { - DCHECK_EQ(expr->args()->numArgs(), 1); - auto argExpr = expr->args()->args()[0]; - auto edgeAlias = extractColNameFromInputPropOrVarPropExpr(argExpr); - if (!edgeAlias.empty()) { - auto it = propsUsed_.edgePropsMap.find(edgeAlias); - if (it == propsUsed_.edgePropsMap.end()) { - propsUsed_.edgePropsMap[edgeAlias] = {}; - } - } + std::transform(funName.begin(), funName.end(), funName.begin(), ::tolower); + if (ignoreFuncs.find(funName) != ignoreFuncs.end()) { return; } @@ -193,6 +268,20 @@ void PropertyTrackerVisitor::visit(FunctionCallExpression *expr) { } } +void PropertyTrackerVisitor::visit(AggregateExpression *expr) { + auto funName = expr->name(); + std::transform(funName.begin(), funName.end(), funName.begin(), ::tolower); + if (funName == "count") { + auto kind = expr->arg()->kind(); + if (kind == Expression::Kind::kConstant || kind == Expression::Kind::kInputProperty || + kind == Expression::Kind::kVarProperty) { + return; + } + } + // count(v.player.age) + expr->arg()->accept(this); +} + void PropertyTrackerVisitor::visit(DestPropertyExpression *expr) { UNUSED(expr); } @@ -253,17 +342,5 @@ void PropertyTrackerVisitor::visit(EdgeExpression *expr) { UNUSED(expr); } -std::string PropertyTrackerVisitor::extractColNameFromInputPropOrVarPropExpr( - const Expression *expr) { - if (expr->kind() == Expression::Kind::kInputProperty) { - auto *inputPropExpr = static_cast(expr); - return inputPropExpr->prop(); - } else if (expr->kind() == Expression::Kind::kVarProperty) { - auto *varPropExpr = static_cast(expr); - return varPropExpr->prop(); - } - return ""; -} - } // namespace graph } // namespace nebula diff --git a/src/graph/visitor/PropertyTrackerVisitor.h b/src/graph/visitor/PropertyTrackerVisitor.h index e8750391656..b7f10507192 100644 --- a/src/graph/visitor/PropertyTrackerVisitor.h +++ b/src/graph/visitor/PropertyTrackerVisitor.h @@ -27,6 +27,9 @@ struct PropertyTracker { Status update(const std::string& oldName, const std::string& newName); bool hasAlias(const std::string& name) const; + void insertVertexProp(const std::string& name, TagID tagId, const std::string& propName); + void insertEdgeProp(const std::string& name, EdgeType type, const std::string& propName); + void insertCols(const std::string& name); }; class PropertyTrackerVisitor : public ExprVisitorImpl { @@ -69,10 +72,10 @@ class PropertyTrackerVisitor : public ExprVisitorImpl { void visit(VertexExpression* expr) override; void visit(EdgeExpression* expr) override; void visit(ColumnExpression* expr) override; - - std::string extractColNameFromInputPropOrVarPropExpr(const Expression* expr); + void visit(AggregateExpression* expr) override; const QueryContext* qctx_{nullptr}; + const int unKnowType_{0}; GraphSpaceID space_; PropertyTracker& propsUsed_; std::string entityAlias_; diff --git a/src/graph/visitor/PrunePropertiesVisitor.cpp b/src/graph/visitor/PrunePropertiesVisitor.cpp index bd50bd7b9ac..59f409cceb1 100644 --- a/src/graph/visitor/PrunePropertiesVisitor.cpp +++ b/src/graph/visitor/PrunePropertiesVisitor.cpp @@ -4,7 +4,7 @@ */ #include "graph/visitor/PrunePropertiesVisitor.h" - +DECLARE_bool(optimize_appendvertices); namespace nebula { namespace graph { @@ -40,45 +40,55 @@ void PrunePropertiesVisitor::visit(Project *node) { void PrunePropertiesVisitor::visitCurrent(Project *node) { // TODO won't use properties of not-root Project - // bool used = used_; - used_ = false; - if (node->columns()) { - const auto &columns = node->columns()->columns(); - auto &colNames = node->colNames(); - std::vector aliasExists(colNames.size(), false); - for (size_t i = 0; i < columns.size(); ++i) { - aliasExists[i] = propsUsed_.hasAlias(colNames[i]); - } - for (size_t i = 0; i < columns.size(); ++i) { + if (!node->columns()) { + return; + } + const auto &columns = node->columns()->columns(); + auto &colNames = node->colNames(); + if (rootNode_) { + for (auto i = 0u; i < columns.size(); ++i) { auto *col = DCHECK_NOTNULL(columns[i]); auto *expr = col->expr(); - auto &alias = colNames[i]; - // If the alias exists, try to rename alias - if (aliasExists[i]) { - if (expr->kind() == Expression::Kind::kInputProperty) { - auto *inputPropExpr = static_cast(expr); - auto &newAlias = inputPropExpr->prop(); - status_ = propsUsed_.update(alias, newAlias); - if (!status_.ok()) { - return; - } - } else if (expr->kind() == Expression::Kind::kVarProperty) { - auto *varPropExpr = static_cast(expr); - auto &newAlias = varPropExpr->prop(); + status_ = extractPropsFromExpr(expr); + if (!status_.ok()) { + return; + } + } + rootNode_ = false; + return; + } + for (auto i = 0u; i < columns.size(); ++i) { + auto *col = DCHECK_NOTNULL(columns[i]); + auto *expr = col->expr(); + auto &alias = colNames[i]; + switch (expr->kind()) { + case Expression::Kind::kVarProperty: + case Expression::Kind::kInputProperty: { + if (propsUsed_.hasAlias(alias)) { + auto *propExpr = static_cast(expr); + auto &newAlias = propExpr->prop(); status_ = propsUsed_.update(alias, newAlias); if (!status_.ok()) { return; } - } else { // eg. "PathBuild[$-.x,$-.__VAR_0,$-.y] AS p" - // How to handle this case? - propsUsed_.colsSet.erase(alias); + } + break; + } + // $-.e[0] as e + // [e IN $-.e WHERE is_edge($e)] AS e + // PathBuild[$-.v,$-.e,$-.v2] AS p + case Expression::Kind::kSubscript: + case Expression::Kind::kPathBuild: + case Expression::Kind::kListComprehension: { + if (propsUsed_.hasAlias(alias)) { status_ = extractPropsFromExpr(expr); if (!status_.ok()) { return; } } - } else { - // Otherwise, extract properties from the column expression + break; + } + default: { status_ = extractPropsFromExpr(expr); if (!status_.ok()) { return; @@ -89,18 +99,29 @@ void PrunePropertiesVisitor::visitCurrent(Project *node) { } void PrunePropertiesVisitor::visit(Aggregate *node) { + rootNode_ = false; visitCurrent(node); status_ = depsPruneProperties(node->dependencies()); } void PrunePropertiesVisitor::visitCurrent(Aggregate *node) { for (auto *groupKey : node->groupKeys()) { + if (groupKey->kind() == Expression::Kind::kVarProperty || + groupKey->kind() == Expression::Kind::kInputProperty || + groupKey->kind() == Expression::Kind::kConstant) { + continue; + } status_ = extractPropsFromExpr(groupKey); if (!status_.ok()) { return; } } for (auto *groupItem : node->groupItems()) { + if (groupItem->kind() == Expression::Kind::kVarProperty || + groupItem->kind() == Expression::Kind::kInputProperty || + groupItem->kind() == Expression::Kind::kConstant) { + continue; + } status_ = extractPropsFromExpr(groupItem); if (!status_.ok()) { return; @@ -108,14 +129,68 @@ void PrunePropertiesVisitor::visitCurrent(Aggregate *node) { } } +void PrunePropertiesVisitor::visit(ScanEdges *node) { + rootNode_ = false; + pruneCurrent(node); + status_ = depsPruneProperties(node->dependencies()); +} + +void PrunePropertiesVisitor::pruneCurrent(ScanEdges *node) { + auto &colNames = node->colNames(); + DCHECK(!colNames.empty()); + auto &edgeAlias = colNames.back(); + auto it = propsUsed_.colsSet.find(edgeAlias); + if (it != propsUsed_.colsSet.end()) { + // all properties are used + return; + } + auto *edgeProps = node->props(); + auto &edgePropsMap = propsUsed_.edgePropsMap; + + auto prunedEdgeProps = std::make_unique>(); + prunedEdgeProps->reserve(edgeProps->size()); + auto edgeAliasIter = edgePropsMap.find(edgeAlias); + + for (auto &edgeProp : *edgeProps) { + auto edgeType = edgeProp.type_ref().value(); + auto &props = edgeProp.props_ref().value(); + EdgeProp newEdgeProp; + newEdgeProp.type_ref() = edgeType; + if (edgeAliasIter == edgePropsMap.end()) { + // only type, dst are used + newEdgeProp.props_ref() = {nebula::kSrc, nebula::kDst, nebula::kType, nebula::kRank}; + } else { + std::unordered_set uniqueProps{ + nebula::kSrc, nebula::kDst, nebula::kType, nebula::kRank}; + std::vector newProps; + auto &usedEdgeProps = edgeAliasIter->second; + auto edgeTypeIter = usedEdgeProps.find(std::abs(edgeType)); + if (edgeTypeIter != usedEdgeProps.end()) { + uniqueProps.insert(edgeTypeIter->second.begin(), edgeTypeIter->second.end()); + } + auto unKnowEdgeIter = usedEdgeProps.find(unKnowType_); + if (unKnowEdgeIter != usedEdgeProps.end()) { + uniqueProps.insert(unKnowEdgeIter->second.begin(), unKnowEdgeIter->second.end()); + } + for (auto &prop : props) { + if (uniqueProps.find(prop) != uniqueProps.end()) { + newProps.emplace_back(prop); + } + } + newEdgeProp.props_ref() = std::move(newProps); + } + prunedEdgeProps->emplace_back(std::move(newEdgeProp)); + } + node->setEdgeProps(std::move(prunedEdgeProps)); +} + void PrunePropertiesVisitor::visit(Traverse *node) { + rootNode_ = false; visitCurrent(node); status_ = depsPruneProperties(node->dependencies()); } void PrunePropertiesVisitor::visitCurrent(Traverse *node) { - bool used = used_; - used_ = false; auto &colNames = node->colNames(); DCHECK_GE(colNames.size(), 2); auto &nodeAlias = colNames[colNames.size() - 2]; @@ -134,32 +209,7 @@ void PrunePropertiesVisitor::visitCurrent(Traverse *node) { return; } } - - if (used) { - // All properties will be used - const auto *vertexProps = node->vertexProps(); - if (vertexProps != nullptr) { - for (const auto &vertexProp : *vertexProps) { - auto tagId = vertexProp.tag_ref().value(); - auto &props = vertexProp.props_ref().value(); - for (const auto &prop : props) { - propsUsed_.vertexPropsMap[nodeAlias][tagId].emplace(prop); - } - } - } - const auto *edgeProps = node->edgeProps(); - if (edgeProps != nullptr) { - for (const auto &edgeProp : *edgeProps) { - auto edgeType = edgeProp.type_ref().value(); - auto &props = edgeProp.props_ref().value(); - for (const auto &prop : props) { - propsUsed_.edgePropsMap[edgeAlias][edgeType].emplace(prop); - } - } - } - } else { - pruneCurrent(node); - } + pruneCurrent(node); } void PrunePropertiesVisitor::pruneCurrent(Traverse *node) { @@ -168,78 +218,94 @@ void PrunePropertiesVisitor::pruneCurrent(Traverse *node) { auto &nodeAlias = colNames[colNames.size() - 2]; auto &edgeAlias = colNames.back(); auto *vertexProps = node->vertexProps(); - if (propsUsed_.colsSet.find(nodeAlias) == propsUsed_.colsSet.end() && vertexProps != nullptr) { - auto it2 = propsUsed_.vertexPropsMap.find(nodeAlias); - if (it2 == propsUsed_.vertexPropsMap.end()) { // nodeAlias is not used + auto &colsSet = propsUsed_.colsSet; + auto &vertexPropsMap = propsUsed_.vertexPropsMap; + auto &edgePropsMap = propsUsed_.edgePropsMap; + + if (colsSet.find(nodeAlias) == colsSet.end()) { + auto aliasIter = vertexPropsMap.find(nodeAlias); + if (aliasIter == vertexPropsMap.end()) { node->setVertexProps(nullptr); } else { - auto prunedVertexProps = std::make_unique>(); - auto &usedVertexProps = it2->second; + auto &usedVertexProps = aliasIter->second; if (usedVertexProps.empty()) { node->setVertexProps(nullptr); - return; - } - prunedVertexProps->reserve(usedVertexProps.size()); - for (auto &vertexProp : *vertexProps) { - auto tagId = vertexProp.tag_ref().value(); - auto &props = vertexProp.props_ref().value(); - auto it3 = usedVertexProps.find(tagId); - if (it3 != usedVertexProps.end()) { - auto &usedProps = it3->second; - VertexProp newVProp; - newVProp.tag_ref() = tagId; + } else { + auto unknowIter = usedVertexProps.find(unKnowType_); + auto prunedVertexProps = std::make_unique>(); + prunedVertexProps->reserve(usedVertexProps.size()); + for (auto &vertexProp : *vertexProps) { + auto tagId = vertexProp.tag_ref().value(); + auto &props = vertexProp.props_ref().value(); + std::unordered_set usedProps; + if (unknowIter != usedVertexProps.end()) { + usedProps.insert(unknowIter->second.begin(), unknowIter->second.end()); + } + auto tagIter = usedVertexProps.find(tagId); + if (tagIter != usedVertexProps.end()) { + usedProps.insert(tagIter->second.begin(), tagIter->second.end()); + } + if (usedProps.empty()) { + continue; + } std::vector newProps; for (auto &prop : props) { if (usedProps.find(prop) != usedProps.end()) { newProps.emplace_back(prop); } } + if (newProps.empty()) { + continue; + } + VertexProp newVProp; + newVProp.tag_ref() = tagId; newVProp.props_ref() = std::move(newProps); prunedVertexProps->emplace_back(std::move(newVProp)); } + node->setVertexProps(std::move(prunedVertexProps)); } - node->setVertexProps(std::move(prunedVertexProps)); } } - static const std::unordered_set reservedEdgeProps = { - nebula::kSrc, nebula::kType, nebula::kRank, nebula::kDst}; auto *edgeProps = node->edgeProps(); - if (propsUsed_.colsSet.find(edgeAlias) == propsUsed_.colsSet.end() && edgeProps != nullptr) { - auto prunedEdgeProps = std::make_unique>(); - prunedEdgeProps->reserve(edgeProps->size()); - auto it2 = propsUsed_.edgePropsMap.find(edgeAlias); - - for (auto &edgeProp : *edgeProps) { - auto edgeType = edgeProp.type_ref().value(); - auto &props = edgeProp.props_ref().value(); - EdgeProp newEProp; - newEProp.type_ref() = edgeType; - std::vector newProps{reservedEdgeProps.begin(), reservedEdgeProps.end()}; - std::unordered_set usedProps; - if (it2 != propsUsed_.edgePropsMap.end()) { - auto &usedEdgeProps = it2->second; - auto it3 = usedEdgeProps.find(std::abs(edgeType)); - if (it3 != usedEdgeProps.end()) { - usedProps = {it3->second.begin(), it3->second.end()}; - } - static const int kUnknownEdgeType = 0; - auto it4 = usedEdgeProps.find(kUnknownEdgeType); - if (it4 != usedEdgeProps.end()) { - usedProps.insert(it4->second.begin(), it4->second.end()); - } + if (colsSet.find(edgeAlias) != colsSet.end()) { + // all edge properties are used + return; + } + auto prunedEdgeProps = std::make_unique>(); + prunedEdgeProps->reserve(edgeProps->size()); + auto edgeAliasIter = edgePropsMap.find(edgeAlias); + + for (auto &edgeProp : *edgeProps) { + auto edgeType = edgeProp.type_ref().value(); + auto &props = edgeProp.props_ref().value(); + EdgeProp newEdgeProp; + newEdgeProp.type_ref() = edgeType; + if (edgeAliasIter == edgePropsMap.end()) { + // only type, dst are used + newEdgeProp.props_ref() = {nebula::kDst, nebula::kRank, nebula::kType}; + } else { + std::unordered_set uniqueProps{nebula::kDst, nebula::kRank, nebula::kType}; + std::vector newProps; + auto &usedEdgeProps = edgeAliasIter->second; + auto edgeTypeIter = usedEdgeProps.find(std::abs(edgeType)); + if (edgeTypeIter != usedEdgeProps.end()) { + uniqueProps.insert(edgeTypeIter->second.begin(), edgeTypeIter->second.end()); + } + auto unKnowEdgeIter = usedEdgeProps.find(unKnowType_); + if (unKnowEdgeIter != usedEdgeProps.end()) { + uniqueProps.insert(unKnowEdgeIter->second.begin(), unKnowEdgeIter->second.end()); } for (auto &prop : props) { - if (reservedEdgeProps.find(prop) == reservedEdgeProps.end() && - usedProps.find(prop) != usedProps.end()) { + if (uniqueProps.find(prop) != uniqueProps.end()) { newProps.emplace_back(prop); } } - newEProp.props_ref() = std::move(newProps); - prunedEdgeProps->emplace_back(std::move(newEProp)); + newEdgeProp.props_ref() = std::move(newProps); } - node->setEdgeProps(std::move(prunedEdgeProps)); + prunedEdgeProps->emplace_back(std::move(newEdgeProp)); } + node->setEdgeProps(std::move(prunedEdgeProps)); } // AppendVertices should be deleted when no properties it pulls are used by the parent node. @@ -249,38 +315,31 @@ void PrunePropertiesVisitor::visit(AppendVertices *node) { } void PrunePropertiesVisitor::visitCurrent(AppendVertices *node) { - bool used = used_; - used_ = false; + if (rootNode_) { + rootNode_ = false; + return; + } auto &colNames = node->colNames(); DCHECK(!colNames.empty()); auto &nodeAlias = colNames.back(); auto it = propsUsed_.colsSet.find(nodeAlias); - if (it != propsUsed_.colsSet.end()) { // All properties are used - // propsUsed_.colsSet.erase(it); + if (it != propsUsed_.colsSet.end()) { + // all properties are used return; } - - if (node->vFilter() != nullptr) { - status_ = extractPropsFromExpr(node->vFilter(), nodeAlias); + if (node->filter() != nullptr) { + status_ = extractPropsFromExpr(node->filter(), nodeAlias); if (!status_.ok()) { return; } } - if (used) { - // All properties will be used - auto *vertexProps = node->props(); - if (vertexProps != nullptr) { - for (const auto &vertexProp : *vertexProps) { - auto tagId = vertexProp.tag_ref().value(); - auto &props = vertexProp.props_ref().value(); - for (const auto &prop : props) { - propsUsed_.vertexPropsMap[nodeAlias][tagId].emplace(prop); - } - } + if (node->vFilter() != nullptr) { + status_ = extractPropsFromExpr(node->vFilter(), nodeAlias); + if (!status_.ok()) { + return; } - } else { - pruneCurrent(node); } + pruneCurrent(node); } void PrunePropertiesVisitor::pruneCurrent(AppendVertices *node) { @@ -288,60 +347,117 @@ void PrunePropertiesVisitor::pruneCurrent(AppendVertices *node) { DCHECK(!colNames.empty()); auto &nodeAlias = colNames.back(); auto *vertexProps = node->props(); - if (vertexProps != nullptr) { - auto prunedVertexProps = std::make_unique>(); - auto it2 = propsUsed_.vertexPropsMap.find(nodeAlias); - if (it2 != propsUsed_.vertexPropsMap.end()) { - auto &usedVertexProps = it2->second; - if (usedVertexProps.empty()) { - node->markDeleted(); - return; - } - prunedVertexProps->reserve(usedVertexProps.size()); - for (auto &vertexProp : *vertexProps) { - auto tagId = vertexProp.tag_ref().value(); - auto &props = vertexProp.props_ref().value(); - auto it3 = usedVertexProps.find(tagId); - if (it3 != usedVertexProps.end()) { - auto &usedProps = it3->second; - VertexProp newVProp; - newVProp.tag_ref() = tagId; - std::vector newProps; - for (auto &prop : props) { - if (usedProps.find(prop) != usedProps.end()) { - newProps.emplace_back(prop); - } - } - newVProp.props_ref() = std::move(newProps); - prunedVertexProps->emplace_back(std::move(newVProp)); - } - } + if (vertexProps == nullptr) { + return; + } + auto prunedVertexProps = std::make_unique>(); + auto &vertexPropsMap = propsUsed_.vertexPropsMap; + auto aliasIter = vertexPropsMap.find(nodeAlias); + if (aliasIter == vertexPropsMap.end()) { + if (FLAGS_optimize_appendvertices) { + node->setVertexProps(nullptr); } else { - node->markDeleted(); - return; + // only get _tag when props is nullptr + auto tagId = vertexProps->front().tag_ref().value(); + VertexProp newVProp; + newVProp.tag_ref() = tagId; + newVProp.props_ref() = {nebula::kTag}; + prunedVertexProps->emplace_back(std::move(newVProp)); + node->setVertexProps(std::move(prunedVertexProps)); + } + return; + } + auto &usedVertexProps = aliasIter->second; + if (usedVertexProps.empty()) { + if (FLAGS_optimize_appendvertices) { + node->setVertexProps(nullptr); + } else { + // only get _tag when props is nullptr + auto tagId = vertexProps->front().tag_ref().value(); + VertexProp newVProp; + newVProp.tag_ref() = tagId; + newVProp.props_ref() = {nebula::kTag}; + prunedVertexProps->emplace_back(std::move(newVProp)); + node->setVertexProps(std::move(prunedVertexProps)); + } + return; + } + auto unknowIter = usedVertexProps.find(unKnowType_); + prunedVertexProps->reserve(usedVertexProps.size()); + for (auto &vertexProp : *vertexProps) { + auto tagId = vertexProp.tag_ref().value(); + auto &props = vertexProp.props_ref().value(); + auto tagIter = usedVertexProps.find(tagId); + std::unordered_set usedProps; + if (unknowIter != usedVertexProps.end()) { + usedProps.insert(unknowIter->second.begin(), unknowIter->second.end()); + } + if (tagIter != usedVertexProps.end()) { + usedProps.insert(tagIter->second.begin(), tagIter->second.end()); + } + if (usedProps.empty()) { + continue; + } + std::vector newProps; + for (auto &prop : props) { + if (usedProps.find(prop) != usedProps.end()) { + newProps.emplace_back(prop); + } + } + if (newProps.empty()) { + continue; } - node->setVertexProps(std::move(prunedVertexProps)); + VertexProp newVProp; + newVProp.tag_ref() = tagId; + newVProp.props_ref() = std::move(newProps); + prunedVertexProps->emplace_back(std::move(newVProp)); } + node->setVertexProps(std::move(prunedVertexProps)); } void PrunePropertiesVisitor::visit(BiJoin *node) { + status_ = depsPruneProperties(node->dependencies()); +} + +void PrunePropertiesVisitor::visit(BiCartesianProduct *node) { + status_ = pruneMultiBranch(node->dependencies()); +} + +void PrunePropertiesVisitor::visit(Union *node) { + status_ = pruneMultiBranch(node->dependencies()); +} + +void PrunePropertiesVisitor::visit(Unwind *node) { visitCurrent(node); status_ = depsPruneProperties(node->dependencies()); } -void PrunePropertiesVisitor::visitCurrent(BiJoin *node) { - for (auto *hashKey : node->hashKeys()) { - status_ = extractPropsFromExpr(hashKey); +void PrunePropertiesVisitor::visitCurrent(Unwind *node) { + const auto &alias = node->alias(); + if (propsUsed_.hasAlias(alias)) { + status_ = extractPropsFromExpr(node->unwindExpr()); if (!status_.ok()) { return; } } - for (auto *probeKey : node->probeKeys()) { - status_ = extractPropsFromExpr(probeKey); - if (!status_.ok()) { - return; - } +} + +Status PrunePropertiesVisitor::pruneMultiBranch(std::vector &dependencies) { + DCHECK_EQ(dependencies.size(), 2); + auto rightPropsUsed = propsUsed_; + auto *leftDep = dependencies.front(); + const_cast(leftDep)->accept(this); + if (!status_.ok()) { + return status_; } + rootNode_ = true; + propsUsed_ = std::move(rightPropsUsed); + auto *rightDep = dependencies.back(); + const_cast(rightDep)->accept(this); + if (!status_.ok()) { + return status_; + } + return Status::OK(); } Status PrunePropertiesVisitor::depsPruneProperties(std::vector &dependencies) { diff --git a/src/graph/visitor/PrunePropertiesVisitor.h b/src/graph/visitor/PrunePropertiesVisitor.h index c6f67e18518..cb14523e1ba 100644 --- a/src/graph/visitor/PrunePropertiesVisitor.h +++ b/src/graph/visitor/PrunePropertiesVisitor.h @@ -55,6 +55,10 @@ class PrunePropertiesVisitor final : public PlanNodeVisitor { // prune properties in Traverse according to the used properties collected previous void pruneCurrent(Traverse *node); + void visit(ScanEdges *node) override; + + void pruneCurrent(ScanEdges *node); + void visit(AppendVertices *node) override; // \param node, the current node to visit // \param used, whether properties in current node are used @@ -63,20 +67,24 @@ class PrunePropertiesVisitor final : public PlanNodeVisitor { void pruneCurrent(AppendVertices *node); void visit(BiJoin *node) override; - // \param node, the current node to visit - // \param used, whether properties in current node are used - void visitCurrent(BiJoin *node); + + void visit(Union *node) override; + void visit(BiCartesianProduct *node) override; + + void visit(Unwind *node) override; + void visitCurrent(Unwind *node); private: Status depsPruneProperties(std::vector &dependencies); + Status pruneMultiBranch(std::vector &dependencies); Status extractPropsFromExpr(const Expression *expr, const std::string &entityAlias = ""); PropertyTracker &propsUsed_; QueryContext *qctx_; GraphSpaceID spaceID_; Status status_; - // force use all properties in current node, e.g. the root node in plan - bool used_{true}; + bool rootNode_{true}; + const int unKnowType_ = 0; }; } // namespace graph diff --git a/src/kvstore/RocksEngine.cpp b/src/kvstore/RocksEngine.cpp index acd740c3dd5..b00128bacc6 100644 --- a/src/kvstore/RocksEngine.cpp +++ b/src/kvstore/RocksEngine.cpp @@ -538,7 +538,7 @@ void RocksEngine::openBackupEngine(GraphSpaceID spaceId) { } } rocksdb::BackupEngine* backupDb; - rocksdb::BackupableDBOptions backupOptions(backupPath_); + rocksdb::BackupEngineOptions backupOptions(backupPath_); backupOptions.backup_log_files = false; auto status = rocksdb::BackupEngine::Open(rocksdb::Env::Default(), backupOptions, &backupDb); CHECK(status.ok()) << status.ToString(); diff --git a/src/kvstore/RocksEngine.h b/src/kvstore/RocksEngine.h index 2de38333efe..17569163c8e 100644 --- a/src/kvstore/RocksEngine.h +++ b/src/kvstore/RocksEngine.h @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include "common/base/Base.h" diff --git a/src/kvstore/raftex/RaftPart.cpp b/src/kvstore/raftex/RaftPart.cpp index 36ba695bb60..faaa73f6a99 100644 --- a/src/kvstore/raftex/RaftPart.cpp +++ b/src/kvstore/raftex/RaftPart.cpp @@ -484,6 +484,12 @@ void RaftPart::stop() { VLOG(1) << idStr_ << "Partition has been stopped"; } +std::pair RaftPart::getTermAndRole() const { + std::lock_guard g(raftLock_); + std::pair res = {term_, role_}; + return res; +} + void RaftPart::cleanWal() { std::lock_guard g(raftLock_); wal()->cleanWAL(committedLogId_); @@ -1462,12 +1468,6 @@ void RaftPart::processAskForVoteRequest(const cpp2::AskForVoteRequest& req, return; } - if (UNLIKELY(status_ == Status::WAITING_SNAPSHOT)) { - VLOG(3) << idStr_ << "The partition is still waiting snapshot"; - resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_WAITING_SNAPSHOT; - return; - } - VLOG(1) << idStr_ << "The partition currently is a " << roleStr(role_) << ", lastLogId " << lastLogId_ << ", lastLogTerm " << lastLogTerm_ << ", committedLogId " << committedLogId_ << ", term " << term_; @@ -1952,18 +1952,19 @@ void RaftPart::processSendSnapshotRequest(const cpp2::SendSnapshotRequest& req, resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_NOT_READY; return; } - // Check leadership - nebula::cpp2::ErrorCode err = verifyLeader(req); - // Set term_ again because it may be modified in verifyLeader - resp.current_term_ref() = term_; - if (err != nebula::cpp2::ErrorCode::SUCCEEDED) { - // Wrong leadership - VLOG(3) << idStr_ << "Will not follow the leader"; - resp.error_code_ref() = err; - return; - } + resp.current_term_ref() = req.get_current_term(); if (status_ != Status::WAITING_SNAPSHOT) { VLOG(2) << idStr_ << "Begin to receive the snapshot"; + // Check leadership + nebula::cpp2::ErrorCode err = verifyLeader(req); + // Set term_ again because it may be modified in verifyLeader + resp.current_term_ref() = term_; + if (err != nebula::cpp2::ErrorCode::SUCCEEDED) { + // Wrong leadership + VLOG(3) << idStr_ << "Will not follow the leader"; + resp.error_code_ref() = err; + return; + } reset(); status_ = Status::WAITING_SNAPSHOT; lastSnapshotCommitId_ = req.get_committed_log_id(); @@ -1978,6 +1979,11 @@ void RaftPart::processSendSnapshotRequest(const cpp2::SendSnapshotRequest& req, resp.error_code_ref() = nebula::cpp2::ErrorCode::E_RAFT_WAITING_SNAPSHOT; return; } + if (term_ > req.get_current_term()) { + VLOG(2) << idStr_ << "leader changed, new term " << term_ + << " but continue to receive snapshot from old leader " << req.get_leader_addr() + << " of term " << req.get_current_term(); + } lastSnapshotRecvDur_.reset(); // TODO(heng): Maybe we should save them into one sst firstly? auto ret = commitSnapshot( @@ -2034,6 +2040,10 @@ void RaftPart::sendHeartbeat() { decltype(hosts_) hosts; { std::lock_guard g(raftLock_); + nebula::cpp2::ErrorCode rc = canAppendLogs(); + if (rc != nebula::cpp2::ErrorCode::SUCCEEDED) { + return; + } currTerm = term_; commitLogId = committedLogId_; prevLogTerm = lastLogTerm_; diff --git a/src/kvstore/raftex/RaftPart.h b/src/kvstore/raftex/RaftPart.h index a495cdf1c20..b2bd3c5a921 100644 --- a/src/kvstore/raftex/RaftPart.h +++ b/src/kvstore/raftex/RaftPart.h @@ -436,6 +436,12 @@ class RaftPart : public std::enable_shared_from_this { using Status = cpp2::Status; using Role = cpp2::Role; + /** + * @brief return term and role + * @return + */ + std::pair getTermAndRole() const; + /** * @brief The str of the RaftPart, used in logging */ diff --git a/src/kvstore/raftex/SnapshotManager.cpp b/src/kvstore/raftex/SnapshotManager.cpp index 0a38eb93e7b..ad2ebfbff44 100644 --- a/src/kvstore/raftex/SnapshotManager.cpp +++ b/src/kvstore/raftex/SnapshotManager.cpp @@ -36,7 +36,13 @@ folly::Future>> SnapshotManager::sendSnapshot( executor_->add([this, p = std::move(p), part, dst]() mutable { auto spaceId = part->spaceId_; auto partId = part->partId_; - auto termId = part->term_; + auto tr = part->getTermAndRole(); + if (tr.second != RaftPart::Role::LEADER) { + VLOG(1) << part->idStr_ << "leader changed, term " << tr.first << ", do not send snapshot to " + << dst; + return; + } + auto termId = tr.first; const auto& localhost = part->address(); accessAllRowsInSnapshot( spaceId, diff --git a/src/parser/Sentence.h b/src/parser/Sentence.h index 18b998a4d65..6afcde73784 100644 --- a/src/parser/Sentence.h +++ b/src/parser/Sentence.h @@ -132,6 +132,7 @@ class Sentence { kShowMetaLeader, kAlterSpace, kClearSpace, + kUnwind, }; Kind kind() const { diff --git a/src/parser/TraverseSentences.cpp b/src/parser/TraverseSentences.cpp index 9114bed6124..5e21a6b3c44 100644 --- a/src/parser/TraverseSentences.cpp +++ b/src/parser/TraverseSentences.cpp @@ -39,6 +39,18 @@ std::string GoSentence::toString() const { return buf; } +std::string UnwindSentence::toString() const { + std::string buf; + buf.reserve(256); + + buf += "UNWIND "; + buf += expr_->toString(); + buf += " AS "; + buf += alias_; + + return buf; +} + LookupSentence::LookupSentence(std::string *from, WhereClause *where, YieldClause *yield) : Sentence(Kind::kLookup), from_(DCHECK_NOTNULL(from)), diff --git a/src/parser/TraverseSentences.h b/src/parser/TraverseSentences.h index f4044e0aae1..54d14713aa4 100644 --- a/src/parser/TraverseSentences.h +++ b/src/parser/TraverseSentences.h @@ -78,6 +78,33 @@ class GoSentence final : public Sentence { std::unique_ptr truncateClause_; }; +class UnwindSentence final : public Sentence { + public: + UnwindSentence(Expression* expr, const std::string& alias) { + expr_ = expr; + alias_ = alias; + kind_ = Kind::kUnwind; + } + + Expression* expr() { + return expr_; + } + + const Expression* expr() const { + return expr_; + } + + const std::string& alias() const { + return alias_; + } + + std::string toString() const override; + + private: + Expression* expr_{nullptr}; + std::string alias_; +}; + class LookupSentence final : public Sentence { public: LookupSentence(std::string* from, WhereClause* where, YieldClause* yield); diff --git a/src/parser/parser.yy b/src/parser/parser.yy index 05e96dfa713..91f5c8e269b 100644 --- a/src/parser/parser.yy +++ b/src/parser/parser.yy @@ -390,7 +390,7 @@ using namespace nebula; %type update_vertex_sentence update_edge_sentence %type download_sentence ingest_sentence -%type traverse_sentence +%type traverse_sentence unwind_sentence %type go_sentence match_sentence lookup_sentence find_path_sentence get_subgraph_sentence %type group_by_sentence order_by_sentence limit_sentence %type fetch_sentence fetch_vertices_sentence fetch_edges_sentence @@ -1673,6 +1673,13 @@ unwind_clause } ; +unwind_sentence + : KW_UNWIND expression KW_AS name_label { + $$ = new UnwindSentence($2, *$4); + delete $4; + } + ; + with_clause : KW_WITH match_return_items match_order_by match_skip match_limit where_clause { $$ = new WithClause($2, $3, $4, $5, $6, false/*distinct*/); @@ -2935,6 +2942,7 @@ traverse_sentence | show_queries_sentence { $$ = $1; } | kill_query_sentence { $$ = $1; } | describe_user_sentence { $$ = $1; } + | unwind_sentence { $$ = $1; } ; piped_sentence diff --git a/src/storage/exec/QueryUtils.h b/src/storage/exec/QueryUtils.h index 599491551fc..4fb86627b12 100644 --- a/src/storage/exec/QueryUtils.h +++ b/src/storage/exec/QueryUtils.h @@ -73,6 +73,9 @@ class QueryUtils final { if (nullType == NullType::UNKNOWN_PROP) { VLOG(1) << "Fail to read prop " << propName; + if (!field) { + return value; + } if (field->hasDefault()) { DefaultValueContext expCtx; ObjectPool pool; diff --git a/src/storage/exec/ScanNode.h b/src/storage/exec/ScanNode.h index a0446203682..c5c5a9d997e 100644 --- a/src/storage/exec/ScanNode.h +++ b/src/storage/exec/ScanNode.h @@ -90,7 +90,10 @@ class ScanVertexPropNode : public QueryNode { } auto vertexId = NebulaKeyUtils::getVertexId(vIdLen, key); if (vertexId != currentVertexId && !currentVertexId.empty()) { - collectOneRow(isIntId, vIdLen, currentVertexId); + ret = collectOneRow(isIntId, vIdLen, currentVertexId); + if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { + return ret; + } } // collect vertex row currentVertexId = vertexId; if (static_cast(resultDataSet_->rowSize()) >= rowLimit) { @@ -100,7 +103,10 @@ class ScanVertexPropNode : public QueryNode { tagNodes_[tagIdIndex->second]->doExecute(key.toString(), value.toString()); } // iterate key if (static_cast(resultDataSet_->rowSize()) < rowLimit) { - collectOneRow(isIntId, vIdLen, currentVertexId); + ret = collectOneRow(isIntId, vIdLen, currentVertexId); + if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { + return ret; + } } cpp2::ScanCursor c; @@ -111,7 +117,9 @@ class ScanVertexPropNode : public QueryNode { return nebula::cpp2::ErrorCode::SUCCEEDED; } - void collectOneRow(bool isIntId, std::size_t vIdLen, const std::string& currentVertexId) { + nebula::cpp2::ErrorCode collectOneRow(bool isIntId, + std::size_t vIdLen, + const std::string& currentVertexId) { List row; nebula::cpp2::ErrorCode ret = nebula::cpp2::ErrorCode::SUCCEEDED; // vertexId is the first column @@ -172,6 +180,7 @@ class ScanVertexPropNode : public QueryNode { tagNode->clear(); } } + return ret; } private: @@ -251,7 +260,10 @@ class ScanEdgePropNode : public QueryNode { } auto value = iter->val(); edgeNodes_[edgeNodeIndex->second]->doExecute(key.toString(), value.toString()); - collectOneRow(isIntId, vIdLen); + ret = collectOneRow(isIntId, vIdLen); + if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) { + return ret; + } } cpp2::ScanCursor c; @@ -262,9 +274,16 @@ class ScanEdgePropNode : public QueryNode { return nebula::cpp2::ErrorCode::SUCCEEDED; } - void collectOneRow(bool isIntId, std::size_t vIdLen) { + nebula::cpp2::ErrorCode collectOneRow(bool isIntId, std::size_t vIdLen) { List row; nebula::cpp2::ErrorCode ret = nebula::cpp2::ErrorCode::SUCCEEDED; + // Usually there is only one edge node, when all of the egdeNodes are invalid (e.g. ttl + // expired), just skip the row. If we don't skip it, there will be a whole line of empty value. + if (!std::any_of(edgeNodes_.begin(), edgeNodes_.end(), [](const auto& edgeNode) { + return edgeNode->valid(); + })) { + return ret; + } for (auto& edgeNode : edgeNodes_) { ret = edgeNode->collectEdgePropsIfValid( [&row, edgeNode = edgeNode.get(), this]( @@ -312,6 +331,7 @@ class ScanEdgePropNode : public QueryNode { for (auto& edgeNode : edgeNodes_) { edgeNode->clear(); } + return ret; } private: diff --git a/src/storage/query/ScanEdgeProcessor.cpp b/src/storage/query/ScanEdgeProcessor.cpp index 1c63165595d..3e95eccdce6 100644 --- a/src/storage/query/ScanEdgeProcessor.cpp +++ b/src/storage/query/ScanEdgeProcessor.cpp @@ -72,6 +72,7 @@ nebula::cpp2::ErrorCode ScanEdgeProcessor::checkAndBuildContexts(const cpp2::Sca return nullptr; } }); + buildEdgeTTLInfo(); return ret; } diff --git a/src/storage/query/ScanVertexProcessor.cpp b/src/storage/query/ScanVertexProcessor.cpp index 7806daebfdd..6ebdf7f2134 100644 --- a/src/storage/query/ScanVertexProcessor.cpp +++ b/src/storage/query/ScanVertexProcessor.cpp @@ -75,6 +75,7 @@ nebula::cpp2::ErrorCode ScanVertexProcessor::checkAndBuildContexts( return nullptr; } }); + buildTagTTLInfo(); return ret; } diff --git a/src/storage/test/GetPropTest.cpp b/src/storage/test/GetPropTest.cpp index 06bf8f5a0bd..91744115c85 100644 --- a/src/storage/test/GetPropTest.cpp +++ b/src/storage/test/GetPropTest.cpp @@ -802,6 +802,57 @@ TEST(GetPropTest, FilterTest) { // } ASSERT_EQ(expected, *resp.props_ref()); } + // invalid vertex filter + { + LOG(INFO) << "GetVertexPropInValue"; + std::vector vertices = {"Tim Duncan"}; + std::vector>> tags; + tags.emplace_back(player, std::vector{"name", "age", "avgScore"}); + Expression* filter = RelationalExpression::makeEQ( + &pool, + TagPropertyExpression::make(&pool, std::to_string(player), "invalid_property"), + ConstantExpression::make(&pool, 42)); + auto req = buildVertexRequest(totalParts, vertices, tags, filter); + + auto* processor = GetPropProcessor::instance(env, nullptr, nullptr); + auto fut = processor->getFuture(); + processor->process(req); + auto resp = std::move(fut).get(); + + ASSERT_EQ(0, (*resp.result_ref()).failed_parts.size()); + nebula::DataSet expected; + expected.colNames = {kVid, "1.name", "1.age", "1.avgScore"}; + ASSERT_EQ(expected, *resp.props_ref()); + } + // invalid edge filter + { + std::vector edgeKeys; + { + cpp2::EdgeKey edgeKey; + edgeKey.src_ref() = "Tim Duncan"; + edgeKey.edge_type_ref() = 101; + edgeKey.ranking_ref() = 1997; + edgeKey.dst_ref() = "Spurs"; + edgeKeys.emplace_back(std::move(edgeKey)); + } + std::vector>> edges; + edges.emplace_back(serve, std::vector{"teamName", "startYear", "endYear"}); + Expression* filter = RelationalExpression::makeEQ( + &pool, + EdgePropertyExpression::make(&pool, std::to_string(serve), "invalid_property"), + ConstantExpression::make(&pool, 42)); + auto req = buildEdgeRequest(totalParts, edgeKeys, edges, filter); + + auto* processor = GetPropProcessor::instance(env, nullptr, nullptr); + auto fut = processor->getFuture(); + processor->process(req); + auto resp = std::move(fut).get(); + + ASSERT_EQ(0, (*resp.result_ref()).failed_parts.size()); + nebula::DataSet expected; + expected.colNames = {"101.teamName", "101.startYear", "101.endYear"}; + ASSERT_EQ(expected, *resp.props_ref()); + } } TEST(GetPropTest, LimitTest) { diff --git a/src/storage/test/ScanEdgeTest.cpp b/src/storage/test/ScanEdgeTest.cpp index 42c41907f9b..012c78ff89b 100644 --- a/src/storage/test/ScanEdgeTest.cpp +++ b/src/storage/test/ScanEdgeTest.cpp @@ -338,6 +338,59 @@ TEST(ScanEdgeTest, FilterTest) { } } +TEST(ScanEdgeTest, TtlTest) { + FLAGS_mock_ttl_col = true; + + fs::TempDir rootPath("/tmp/GetNeighborsTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto totalParts = cluster.getTotalParts(); + ASSERT_EQ(true, QueryTestUtils::mockVertexData(env, totalParts)); + ASSERT_EQ(true, QueryTestUtils::mockEdgeData(env, totalParts)); + + EdgeType serve = 101; + + { + LOG(INFO) << "Scan one edge with some properties in one batch"; + size_t totalRowCount = 0; + auto edge = std::make_pair( + serve, + std::vector{kSrc, kType, kRank, kDst, "teamName", "startYear", "endYear"}); + for (PartitionID partId = 1; partId <= totalParts; partId++) { + auto req = buildRequest({partId}, {""}, {edge}); + auto* processor = ScanEdgeProcessor::instance(env, nullptr); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + + ASSERT_EQ(0, resp.result.failed_parts.size()); + ASSERT_FALSE(resp.get_props()->rows.empty()); + checkResponse(*resp.props_ref(), edge, edge.second.size(), totalRowCount); + } + CHECK_EQ(mock::MockData::serves_.size(), totalRowCount); + } + sleep(FLAGS_mock_ttl_duration + 1); + { + LOG(INFO) << "TTL expired, same request but no data returned"; + auto edge = std::make_pair( + serve, + std::vector{kSrc, kType, kRank, kDst, "teamName", "startYear", "endYear"}); + for (PartitionID partId = 1; partId <= totalParts; partId++) { + auto req = buildRequest({partId}, {""}, {edge}); + auto* processor = ScanEdgeProcessor::instance(env, nullptr); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + + ASSERT_EQ(0, resp.result.failed_parts.size()); + ASSERT_TRUE(resp.get_props()->rows.empty()); + } + } + + FLAGS_mock_ttl_col = false; +} + } // namespace storage } // namespace nebula diff --git a/src/storage/test/ScanVertexTest.cpp b/src/storage/test/ScanVertexTest.cpp index 61f9e006cea..294d02fe69d 100644 --- a/src/storage/test/ScanVertexTest.cpp +++ b/src/storage/test/ScanVertexTest.cpp @@ -501,6 +501,56 @@ TEST(ScanVertexTest, FilterTest) { } } +TEST(ScanVertexTest, TtlTest) { + FLAGS_mock_ttl_col = true; + + fs::TempDir rootPath("/tmp/ScanVertexTest.XXXXXX"); + mock::MockCluster cluster; + cluster.initStorageKV(rootPath.path()); + auto* env = cluster.storageEnv_.get(); + auto totalParts = cluster.getTotalParts(); + ASSERT_EQ(true, QueryTestUtils::mockVertexData(env, totalParts)); + ASSERT_EQ(true, QueryTestUtils::mockEdgeData(env, totalParts)); + + TagID player = 1; + + { + LOG(INFO) << "Scan one tag with some properties in one batch"; + size_t totalRowCount = 0; + auto tag = + std::make_pair(player, std::vector{kVid, kTag, "name", "age", "avgScore"}); + for (PartitionID partId = 1; partId <= totalParts; partId++) { + auto req = buildRequest({partId}, {""}, {tag}); + auto* processor = ScanVertexProcessor::instance(env, nullptr); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + + ASSERT_EQ(0, resp.result.failed_parts.size()); + checkResponse(*resp.props_ref(), tag, tag.second.size() + 1 /* kVid */, totalRowCount); + } + CHECK_EQ(mock::MockData::players_.size(), totalRowCount); + } + sleep(FLAGS_mock_ttl_duration + 1); + { + LOG(INFO) << "TTL expired, same request but no data returned"; + auto tag = + std::make_pair(player, std::vector{kVid, kTag, "name", "age", "avgScore"}); + for (PartitionID partId = 1; partId <= totalParts; partId++) { + auto req = buildRequest({partId}, {""}, {tag}); + auto* processor = ScanVertexProcessor::instance(env, nullptr); + auto f = processor->getFuture(); + processor->process(req); + auto resp = std::move(f).get(); + + ASSERT_EQ(0, resp.result.failed_parts.size()); + ASSERT_TRUE(resp.get_props()->rows.empty()); + } + } + + FLAGS_mock_ttl_col = false; +} + } // namespace storage } // namespace nebula diff --git a/tests/tck/features/expression/NotIn.feature b/tests/tck/features/expression/NotIn.feature index 6fce6111f2c..60fe563f379 100644 --- a/tests/tck/features/expression/NotIn.feature +++ b/tests/tck/features/expression/NotIn.feature @@ -51,6 +51,27 @@ Feature: Not In Expression | r | | false | + Scenario: Match Not In Set + Given a graph with space named "nba" + When executing query: + """ + match p0 = (n0)<-[e0:`like`|`teammate`|`teammate`]->(n1) + where id(n0) == "Suns" + and not (e0.like.likeness in [e0.teammate.end_year, ( e0.teammate.start_year ) ] ) + or not (( "" ) not ends with ( "" + "" + "" )) + and ("" not in ( "" + "" + "" + "" )) + or (e0.teammate.start_year > ( e0.teammate.end_year )) + and (( ( ( e0.like.likeness ) ) ) / e0.teammate.start_year > + e0.teammate.start_year) + or (e0.like.likeness*e0.teammate.start_year%e0.teammate.end_year+ + ( ( e0.teammate.start_year ) ) > e0.teammate.end_year) + or (( ( ( ( e0.teammate.end_year ) ) ) ) in [9.8978784E7 ] ) + return e0.like.likeness, e0.teammate.start_year, e0.teammate.start_year, + e0.teammate.end_year, e0.teammate.end_year + limit 91 + """ + Then a SemanticError should be raised at runtime: Type error `("" NOT IN "")' + Scenario: Using NOT IN list in GO Given a graph with space named "nba" When executing query: diff --git a/tests/tck/features/lookup/LookUp.feature b/tests/tck/features/lookup/LookUp.feature index 2c54e6fe452..769d1503148 100644 --- a/tests/tck/features/lookup/LookUp.feature +++ b/tests/tck/features/lookup/LookUp.feature @@ -71,6 +71,13 @@ Feature: LookUpTest_Vid_String Then the result should be, in any order: | src | dst | rank | | "200" | "201" | 0 | + When executing query: + """ + LOOKUP ON lookup_edge_1 WHERE lookup_edge_1.col2 IN [201] and lookup_edge_1.col2>3 YIELD src(edge) as src, dst(Edge) as dst, rank(edge) as rank + """ + Then the result should be, in any order: + | src | dst | rank | + | "200" | "201" | 0 | When executing query: """ LOOKUP ON lookup_edge_2 WHERE lookup_edge_2.col1 == 200 YIELD edge as e diff --git a/tests/tck/features/lookup/LookUpLimit.feature b/tests/tck/features/lookup/LookUpLimit.feature index d66456c4408..684a5979bd7 100644 --- a/tests/tck/features/lookup/LookUpLimit.feature +++ b/tests/tck/features/lookup/LookUpLimit.feature @@ -17,7 +17,6 @@ Feature: Push Limit down IndexScan Rule | /[a-zA-Z ']+/ | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | Sort | 6 | | | 6 | Project | 7 | | | 7 | Limit | 8 | {"count": "2"} | @@ -33,7 +32,6 @@ Feature: Push Limit down IndexScan Rule | /[a-zA-Z ']+/ | /[a-zA-Z ']+/ | /\d+/ | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | Sort | 6 | | | 6 | Project | 7 | | | 7 | Limit | 8 | {"count": "2"} | @@ -49,7 +47,6 @@ Feature: Push Limit down IndexScan Rule | /[a-zA-Z ']+/ | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | Sort | 6 | | | 6 | Project | 7 | | | 7 | Limit | 8 | {"count": "2"} | @@ -65,7 +62,6 @@ Feature: Push Limit down IndexScan Rule | /[a-zA-Z ']+/ | /[a-zA-Z ']+/ | /\d+/ | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | Sort | 6 | | | 6 | Project | 7 | | | 7 | Limit | 8 | {"count": "2"} | @@ -84,7 +80,6 @@ Feature: Push Limit down IndexScan Rule | /[a-zA-Z ']+/ | And the execution plan should be: | id | name | dependencies | operator info | - | 3 | DataCollect | 4 | | | 4 | Sort | 5 | | | 5 | Project | 7 | | | 7 | Limit | 8 | | @@ -101,7 +96,6 @@ Feature: Push Limit down IndexScan Rule | /[a-zA-Z ']+/ | /[a-zA-Z ']+/ | /\d+/ | And the execution plan should be: | id | name | dependencies | operator info | - | 3 | DataCollect | 4 | | | 4 | Sort | 5 | | | 5 | Project | 7 | | | 7 | Limit | 8 | | @@ -118,7 +112,6 @@ Feature: Push Limit down IndexScan Rule | /[a-zA-Z ']+/ | And the execution plan should be: | id | name | dependencies | operator info | - | 3 | DataCollect | 4 | | | 4 | Sort | 5 | | | 5 | Project | 7 | | | 7 | Limit | 8 | | @@ -135,7 +128,6 @@ Feature: Push Limit down IndexScan Rule | /[a-zA-Z ']+/ | /[a-zA-Z ']+/ | /\d+/ | And the execution plan should be: | id | name | dependencies | operator info | - | 3 | DataCollect | 4 | | | 4 | Sort | 5 | | | 5 | Project | 7 | | | 7 | Limit | 8 | | diff --git a/tests/tck/features/lookup/LookUpTopN.feature b/tests/tck/features/lookup/LookUpTopN.feature index c5aabc4463f..0f4ee041546 100644 --- a/tests/tck/features/lookup/LookUpTopN.feature +++ b/tests/tck/features/lookup/LookUpTopN.feature @@ -17,14 +17,12 @@ Feature: Push TopN down IndexScan Rule | "Aron Baynes" | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | {"count": "2"} | | 6 | Project | 7 | | | 7 | TagIndexFullScan | 0 | {"limit": "9223372036854775807" } | | 0 | Start | | | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | | | 6 | Project | 7 | | | 7 | TagIndexFullScan | 0 | {"orderBy": "[]"} | @@ -39,14 +37,12 @@ Feature: Push TopN down IndexScan Rule | "DeAndre Jordan" | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | {"count": "2"} | | 6 | Project | 7 | | | 7 | TagIndexPrefixScan | 0 | {"limit": "9223372036854775807" } | | 0 | Start | | | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | | | 6 | Project | 7 | | | 7 | TagIndexPrefixScan | 0 | {"orderBy": "[]"} | @@ -61,14 +57,12 @@ Feature: Push TopN down IndexScan Rule | "Tim Duncan" | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | {"count": "2"} | | 6 | Project | 7 | | | 7 | TagIndexRangeScan | 0 | {"limit": "9223372036854775807" } | | 0 | Start | | | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | | | 6 | Project | 7 | | | 7 | TagIndexRangeScan | 0 | {"orderBy": "[]"} | @@ -83,14 +77,12 @@ Feature: Push TopN down IndexScan Rule | "Aron Baynes" | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | {"count": "2"} | | 6 | Project | 7 | | | 7 | TagIndexFullScan | 0 | {"limit": "9223372036854775807" } | | 0 | Start | | | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | | | 6 | Project | 7 | | | 7 | TagIndexFullScan | 0 | {"orderBy": "[]"} | @@ -105,14 +97,12 @@ Feature: Push TopN down IndexScan Rule | "Aron Baynes" | "Tim Duncan" | 0 | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | {"count": "2"} | | 6 | Project | 7 | | | 7 | EdgeIndexFullScan | 0 | {"limit": "9223372036854775807" } | | 0 | Start | | | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | | | 6 | Project | 7 | | | 7 | EdgeIndexFullScan | 0 | {"orderBy": "[]"} | @@ -127,14 +117,12 @@ Feature: Push TopN down IndexScan Rule | "Carmelo Anthony" | "Chris Paul" | 0 | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | {"count": "2"} | | 6 | Project | 7 | | | 7 | EdgeIndexPrefixScan | 0 | {"limit": "9223372036854775807" } | | 0 | Start | | | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | | | 6 | Project | 7 | | | 7 | EdgeIndexPrefixScan | 0 | {"orderBy": "[]"} | @@ -149,14 +137,12 @@ Feature: Push TopN down IndexScan Rule | "Dejounte Murray" | "Chris Paul" | 0 | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | {"count": "2"} | | 6 | Project | 7 | | | 7 | EdgeIndexRangeScan | 0 | {"limit": "9223372036854775807" } | | 0 | Start | | | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | | | 6 | Project | 7 | | | 7 | EdgeIndexRangeScan | 0 | {"orderBy": "[]"} | @@ -173,7 +159,6 @@ Feature: Push TopN down IndexScan Rule | "Aron Baynes" | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | {"count": "2"} | | 6 | Project | 7 | | | 7 | TagIndexFullScan | 0 | {"limit": "2" } | @@ -188,7 +173,6 @@ Feature: Push TopN down IndexScan Rule | "Kevin Durant" | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | {"count": "2"} | | 6 | Project | 7 | | | 7 | TagIndexPrefixScan | 0 | {"limit": "2" } | @@ -203,7 +187,6 @@ Feature: Push TopN down IndexScan Rule | "DeAndre Jordan" | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | {"count": "2"} | | 6 | Project | 7 | | | 7 | TagIndexPrefixScan | 0 | {"limit": "2" } | @@ -218,7 +201,6 @@ Feature: Push TopN down IndexScan Rule | "Tim Duncan" | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | {"count": "2"} | | 6 | Project | 7 | | | 7 | TagIndexRangeScan | 0 | {"limit": "2" } | @@ -233,7 +215,6 @@ Feature: Push TopN down IndexScan Rule | -1 | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | {"count": "2"} | | 6 | Project | 7 | | | 7 | EdgeIndexFullScan | 0 | {"limit": "2" } | @@ -248,7 +229,6 @@ Feature: Push TopN down IndexScan Rule | 100 | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | {"count": "2"} | | 6 | Project | 7 | | | 7 | EdgeIndexFullScan | 0 | {"limit": "2" } | @@ -263,7 +243,6 @@ Feature: Push TopN down IndexScan Rule | 90 | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | {"count": "2"} | | 6 | Project | 7 | | | 7 | EdgeIndexPrefixScan | 0 | {"limit": "2" } | @@ -278,7 +257,6 @@ Feature: Push TopN down IndexScan Rule | 95 | And the execution plan should be: | id | name | dependencies | operator info | - | 4 | DataCollect | 5 | | | 5 | TopN | 6 | {"count": "2"} | | 6 | Project | 7 | | | 7 | EdgeIndexRangeScan | 0 | {"limit": "2" } | diff --git a/tests/tck/features/match/PipeAndVariable.feature b/tests/tck/features/match/PipeAndVariable.feature index 8a0f8cfb1b2..68abc31edc2 100644 --- a/tests/tck/features/match/PipeAndVariable.feature +++ b/tests/tck/features/match/PipeAndVariable.feature @@ -79,26 +79,108 @@ Feature: Pipe or use variable to store the lookup results | true | 42 | Scenario: mixed usage of cypher and ngql + When executing query: + """ + YIELD ['Tim Duncan', 'Tony Parker'] AS a + | UNWIND $-.a AS b + | GO FROM $-.b OVER like YIELD edge AS e + """ + Then the result should be, in any order: + | e | + | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | + | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | + | [:like "Tony Parker"->"LaMarcus Aldridge" @0 {likeness: 90}] | + | [:like "Tony Parker"->"Manu Ginobili" @0 {likeness: 95}] | + | [:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}] | + When executing query: + """ + YIELD {a:1, b:['Tim Duncan', 'Tony Parker'], c:'Tim Duncan'} AS a + | YIELD $-.a.b AS b + | UNWIND $-.b AS c + | GO FROM $-.c OVER like YIELD edge AS e + """ + Then the result should be, in any order: + | e | + | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | + | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | + | [:like "Tony Parker"->"LaMarcus Aldridge" @0 {likeness: 90}] | + | [:like "Tony Parker"->"Manu Ginobili" @0 {likeness: 95}] | + | [:like "Tony Parker"->"Tim Duncan" @0 {likeness: 95}] | + When executing query: + """ + YIELD {a:1, b:['Tim Duncan', 'Tony Parker'], c:'Tim Duncan'} AS a + | YIELD $-.a.c AS b + | UNWIND $-.b AS c + | GO FROM $-.c OVER like YIELD edge AS e + """ + Then the result should be, in any order: + | e | + | [:like "Tim Duncan"->"Manu Ginobili" @0 {likeness: 95}] | + | [:like "Tim Duncan"->"Tony Parker" @0 {likeness: 95}] | When executing query: """ LOOKUP ON player WHERE player.name == 'Tim Duncan' YIELD player.age as age, id(vertex) as vid - | UNWIND $-.vid as a RETURN a + | UNWIND $-.vid as a | YIELD $-.a AS a """ - Then a SyntaxError should be raised at runtime: syntax error near `UNWIND' + Then the result should be, in any order, with relax comparison: + | a | + | "Tim Duncan" | When executing query: """ GET SUBGRAPH 2 STEPS FROM "Tim Duncan" BOTH like YIELD edges as e - | UNWIND $-.e as a RETURN a - """ - Then a SyntaxError should be raised at runtime: syntax error near `UNWIND' + | UNWIND $-.e as a | YIELD $-.a AS a + """ + Then the result should be, in any order, with relax comparison: + | a | + | [:like "Aron Baynes"->"Tim Duncan" @0 {}] | + | [:like "Boris Diaw"->"Tim Duncan" @0 {}] | + | [:like "Danny Green"->"Tim Duncan" @0 {}] | + | [:like "Dejounte Murray"->"Tim Duncan" @0 {}] | + | [:like "LaMarcus Aldridge"->"Tim Duncan" @0 {}] | + | [:like "Manu Ginobili"->"Tim Duncan" @0 {}] | + | [:like "Marco Belinelli"->"Tim Duncan" @0 {}] | + | [:like "Shaquille O'Neal"->"Tim Duncan" @0 {}] | + | [:like "Tiago Splitter"->"Tim Duncan" @0 {}] | + | [:like "Tony Parker"->"Tim Duncan" @0 {}] | + | [:like "Tim Duncan"->"Manu Ginobili" @0 {}] | + | [:like "Tim Duncan"->"Tony Parker" @0 {}] | + | [:like "Damian Lillard"->"LaMarcus Aldridge" @0 {}] | + | [:like "Rudy Gay"->"LaMarcus Aldridge" @0 {}] | + | [:like "LaMarcus Aldridge"->"Tony Parker" @0 {}] | + | [:like "Boris Diaw"->"Tony Parker" @0 {}] | + | [:like "Dejounte Murray"->"Chris Paul" @0 {}] | + | [:like "Dejounte Murray"->"Danny Green" @0 {}] | + | [:like "Dejounte Murray"->"James Harden" @0 {}] | + | [:like "Dejounte Murray"->"Kevin Durant" @0 {}] | + | [:like "Dejounte Murray"->"Kyle Anderson" @0 {}] | + | [:like "Dejounte Murray"->"LeBron James" @0 {}] | + | [:like "Dejounte Murray"->"Manu Ginobili" @0 {}] | + | [:like "Dejounte Murray"->"Marco Belinelli" @0 {}] | + | [:like "Dejounte Murray"->"Russell Westbrook" @0 {}] | + | [:like "Dejounte Murray"->"Tony Parker" @0 {}] | + | [:like "Danny Green"->"LeBron James" @0 {}] | + | [:like "Danny Green"->"Marco Belinelli" @0 {}] | + | [:like "Marco Belinelli"->"Danny Green" @0 {}] | + | [:like "Marco Belinelli"->"Tony Parker" @0 {}] | + | [:like "Yao Ming"->"Shaquille O'Neal" @0 {}] | + | [:like "Shaquille O'Neal"->"JaVale McGee" @0 {}] | + | [:like "Tiago Splitter"->"Manu Ginobili" @0 {}] | + | [:like "Tony Parker"->"LaMarcus Aldridge" @0 {}] | + | [:like "Tony Parker"->"Manu Ginobili" @0 {}] | + | [:like "James Harden"->"Russell Westbrook" @0 {}] | + | [:like "Chris Paul"->"LeBron James" @0 {}] | + | [:like "Russell Westbrook"->"James Harden" @0 {}] | When executing query: """ - FIND SHORTEST PATH FROM "Tim Duncan" TO "Yao Ming" OVER like YIELD path as p - | UNWIND $-.p as a RETURN a + FIND SHORTEST PATH FROM "Tim Duncan" TO "Tony Parker" OVER like YIELD path as p + | YIELD nodes($-.p) AS nodes | UNWIND $-.nodes AS a | YIELD $-.a AS a """ - Then a SyntaxError should be raised at runtime: syntax error near `UNWIND' + Then the result should be, in any order, with relax comparison: + | a | + | ("Tim Duncan") | + | ("Tony Parker") | When executing query: """ GO 2 STEPS FROM "Tim Duncan" OVER * YIELD dst(edge) as id diff --git a/tests/tck/features/match/Unwind.feature b/tests/tck/features/match/Unwind.feature index 8d9b865ce46..40e4893e853 100644 --- a/tests/tck/features/match/Unwind.feature +++ b/tests/tck/features/match/Unwind.feature @@ -146,7 +146,7 @@ Feature: Unwind clause WITH DISTINCT vid RETURN collect(vid) as vids """ - Then a SyntaxError should be raised at runtime: syntax error near `UNWIND' + Then a SyntaxError should be raised at runtime: syntax error near `WITH' When executing query: """ MATCH (a:player {name:"Tim Duncan"}) - [e:like] -> (b) diff --git a/tests/tck/features/optimizer/CollapseProjectRule.feature b/tests/tck/features/optimizer/CollapseProjectRule.feature index 1e52fcabfa0..924b61635ba 100644 --- a/tests/tck/features/optimizer/CollapseProjectRule.feature +++ b/tests/tck/features/optimizer/CollapseProjectRule.feature @@ -95,7 +95,6 @@ Feature: Collapse Project Rule | 29 | "Dejounte Murray" | And the execution plan should be: | id | name | dependencies | operator info | - | 11 | DataCollect | 10 | | | 10 | Dedup | 14 | | | 14 | Project | 12 | | | 12 | Filter | 6 | | diff --git a/tests/tck/features/optimizer/PrunePropertiesRule.feature b/tests/tck/features/optimizer/PrunePropertiesRule.feature index 468b6b8b69e..16f43e5455c 100644 --- a/tests/tck/features/optimizer/PrunePropertiesRule.feature +++ b/tests/tck/features/optimizer/PrunePropertiesRule.feature @@ -3,12 +3,10 @@ # This source code is licensed under Apache 2.0 License. Feature: Prune Properties rule - Background: - Given a graph with space named "nba" - # The schema id is not fixed in standalone cluster, so we skip it @distonly Scenario: Single Match + Given a graph with space named "nba" When profiling query: """ MATCH p = (v:player{name: "Tony Parker"})-[e:like]->(v2) @@ -21,12 +19,12 @@ Feature: Prune Properties rule | 33 | | 41 | And the execution plan should be: - | id | name | dependencies | operator info | - | 8 | Project | 4 | | - | 4 | AppendVertices | 3 | { "props": "[{\"props\":[\"age\"],\"tagId\":9}]" } | - | 3 | Traverse | 7 | { "vertexProps": "", "edgeProps": "[{\"props\":[\"_dst\", \"_rank\", \"_type\", \"_src\"],\"type\":3}]" } | - | 7 | IndexScan | 2 | | - | 2 | Start | | | + | id | name | dependencies | operator info | + | 8 | Project | 4 | | + | 4 | AppendVertices | 3 | { "props": "[{\"props\":[\"age\"],\"tagId\":9}]" } | + | 3 | Traverse | 7 | { "vertexProps": "", "edgeProps": "[{\"props\":[\"_dst\", \"_rank\", \"_type\"],\"type\":3}]" } | + | 7 | IndexScan | 2 | | + | 2 | Start | | | When profiling query: """ MATCH p = (v:player{name: "Tony Parker"})-[e:like]->(v2) @@ -38,12 +36,12 @@ Feature: Prune Properties rule | "Tony Parker" | | "Tony Parker" | And the execution plan should be: - | id | name | dependencies | operator info | - | 8 | Project | 4 | | - | 4 | AppendVertices | 3 | { "props": "[{\"tagId\": 9, \"props\": [\"name\", \"age\", \"_tag\"]}, {\"props\": [\"name\", \"speciality\", \"_tag\"], \"tagId\": 8}, {\"tagId\": 10, \"props\": [\"name\", \"_tag\"]}]"} | - | 3 | Traverse | 7 | { "vertexProps": "[{\"props\":[\"name\"],\"tagId\":9}]", "edgeProps": "[{\"props\":[\"_dst\", \"_rank\", \"_type\", \"_src\"],\"type\":3}]" } | - | 7 | IndexScan | 2 | | - | 2 | Start | | | + | id | name | dependencies | operator info | + | 8 | Project | 4 | | + | 4 | AppendVertices | 3 | { "props": "[{\"tagId\": 9, \"props\": [\"_tag\"]}]"} | + | 3 | Traverse | 7 | { "vertexProps": "[{\"props\":[\"name\"],\"tagId\":9}]", "edgeProps": "[{\"props\":[\"_dst\", \"_rank\", \"_type\"],\"type\":3}]" } | + | 7 | IndexScan | 2 | | + | 2 | Start | | | When profiling query: """ MATCH p = (v:player{name: "Tony Parker"})-[e:like]-(v2) @@ -60,12 +58,12 @@ Feature: Prune Properties rule | ("Tony Parker" :player{age: 36, name: "Tony Parker"}) | | ("Tony Parker" :player{age: 36, name: "Tony Parker"}) | And the execution plan should be: - | id | name | dependencies | operator info | - | 8 | Project | 4 | | - | 4 | AppendVertices | 3 | { "props": "[{\"tagId\": 9, \"props\": [\"name\", \"age\", \"_tag\"]}, {\"props\": [\"name\", \"speciality\", \"_tag\"], \"tagId\": 8}, {\"tagId\": 10, \"props\": [\"name\", \"_tag\"]}]" } | - | 3 | Traverse | 7 | { "vertexProps": "[{\"props\": [\"name\", \"age\", \"_tag\"], \"tagId\": 9}, {\"props\": [\"name\", \"speciality\", \"_tag\"], \"tagId\": 8}, {\"tagId\": 10, \"props\": [\"name\", \"_tag\"]}]", "edgeProps": "[{\"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"], \"type\": -3}, {\"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"], \"type\": 3}]" } | - | 7 | IndexScan | 2 | | - | 2 | Start | | | + | id | name | dependencies | operator info | + | 8 | Project | 4 | | + | 4 | AppendVertices | 3 | { "props": "[{\"tagId\": 9, \"props\": [\"_tag\"]} ]" } | + | 3 | Traverse | 7 | { "vertexProps": "[{\"props\": [\"name\", \"age\", \"_tag\"], \"tagId\": 9}, {\"props\": [\"name\", \"speciality\", \"_tag\"], \"tagId\": 8}, {\"tagId\": 10, \"props\": [\"name\", \"_tag\"]}]", "edgeProps": "[{\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": -3}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": 3}]" } | + | 7 | IndexScan | 2 | | + | 2 | Start | | | When profiling query: """ MATCH p = (v:player{name: "Tony Parker"})-[e:like]->(v2) @@ -80,7 +78,7 @@ Feature: Prune Properties rule | id | name | dependencies | operator info | | 8 | Project | 4 | | | 4 | AppendVertices | 3 | { "props": "[{\"props\":[\"name\", \"age\", \"_tag\"],\"tagId\":9}, {\"props\":[\"name\", \"speciality\", \"_tag\"], \"tagId\":8}, {\"props\":[\"name\", \"_tag\"],\"tagId\":10}]" } | - | 3 | Traverse | 7 | { "vertexProps": "", "edgeProps": "[{\"props\":[\"_dst\", \"_rank\", \"_type\", \"_src\"],\"type\":3}]" } | + | 3 | Traverse | 7 | { "vertexProps": "", "edgeProps": "[{\"props\":[\"_dst\", \"_rank\", \"_type\"],\"type\":3}]" } | | 7 | IndexScan | 2 | | | 2 | Start | | | # The rule will not take affect in this case because it returns the whole path @@ -110,12 +108,12 @@ Feature: Prune Properties rule | "like" | | "like" | And the execution plan should be: - | id | name | dependencies | operator info | - | 8 | Project | 4 | | - | 4 | AppendVertices | 3 | { "props": "[{\"props\":[\"name\", \"age\", \"_tag\"],\"tagId\":9}, {\"props\":[\"name\", \"speciality\", \"_tag\"], \"tagId\":8}, {\"props\":[\"name\", \"_tag\"],\"tagId\":10}]" } | - | 3 | Traverse | 7 | { "vertexProps": "", "edgeProps": "[{\"props\":[\"_dst\", \"_rank\", \"_type\", \"_src\"],\"type\":3}]" } | - | 7 | IndexScan | 2 | | - | 2 | Start | | | + | id | name | dependencies | operator info | + | 8 | Project | 4 | | + | 4 | AppendVertices | 3 | { "props": "[{\"props\":[\"_tag\"],\"tagId\":9} ]" } | + | 3 | Traverse | 7 | { "vertexProps": "", "edgeProps": "[{\"props\":[\"_dst\", \"_rank\", \"_type\"],\"type\":3}]" } | + | 7 | IndexScan | 2 | | + | 2 | Start | | | When executing query: """ MATCH (v:player{name: "Tony Parker"})-[:like]-(v2)--(v3) @@ -151,6 +149,7 @@ Feature: Prune Properties rule # The schema id is not fixed in standalone cluster, so we skip it @distonly Scenario: Multi Path Patterns + Given a graph with space named "nba" When profiling query: """ MATCH (m)-[]-(n), (n)-[]-(l) WHERE id(m)=="Tim Duncan" @@ -171,21 +170,20 @@ Feature: Prune Properties rule | "Tim Duncan" | "Boris Diaw" | "Suns" | | "Tim Duncan" | "Boris Diaw" | "Tim Duncan" | And the execution plan should be: - | id | name | dependencies | operator info | - | 15 | DataCollect | 16 | | - | 16 | TopN | 12 | | - | 12 | Project | 9 | | - | 9 | BiInnerJoin | 22, 23 | | - | 22 | Project | 5 | | - | 5 | AppendVertices | 4 | { "props": "[{\"props\":[\"name\"],\"tagId\":9}]" } | - | 4 | Traverse | 2 | { "vertexProps": "[{\"props\":[\"name\"],\"tagId\":9}]" } | - | 2 | Dedup | 1 | | - | 1 | PassThrough | 3 | | - | 3 | Start | | | - | 23 | Project | 8 | | - | 8 | AppendVertices | 7 | { "props": "[{\"tagId\":9,\"props\":[\"name\"]}, {\"tagId\":10,\"props\":[\"name\"]}]" } | - | 7 | Traverse | 6 | { "vertexProps": "[{\"tagId\":9,\"props\":[\"name\"]}]", "edgeProps": "[{\"type\": -5, \"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"]}, {\"type\": 5, \"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"]}, {\"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"], \"type\": -3}, {\"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"], \"type\": 3}, {\"type\": -4, \"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"]}, {\"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"], \"type\": 4}]" } | - | 6 | Argument | | | + | id | name | dependencies | operator info | + | 16 | TopN | 12 | | + | 12 | Project | 9 | | + | 9 | BiInnerJoin | 22, 23 | | + | 22 | Project | 5 | | + | 5 | AppendVertices | 4 | { "props": "[{\"props\":[\"name\"],\"tagId\":9}]" } | + | 4 | Traverse | 2 | { "vertexProps": "[{\"props\":[\"name\"],\"tagId\":9}]" } | + | 2 | Dedup | 1 | | + | 1 | PassThrough | 3 | | + | 3 | Start | | | + | 23 | Project | 8 | | + | 8 | AppendVertices | 7 | { "props": "[{\"tagId\":9,\"props\":[\"name\"]}, {\"tagId\":10,\"props\":[\"name\"]}]" } | + | 7 | Traverse | 6 | { "vertexProps": "[{\"tagId\":9,\"props\":[\"name\"]}]", "edgeProps": "[{\"type\": -5, \"props\": [\"_dst\", \"_rank\", \"_type\"]}, {\"type\": 5, \"props\": [\"_dst\", \"_rank\", \"_type\"]}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": -3}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": 3}, {\"type\": -4, \"props\": [\"_dst\", \"_rank\", \"_type\"]}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": 4}]" } | + | 6 | Argument | | | When profiling query: """ MATCH (m)-[]-(n), (n)-[]-(l), (l)-[]-(h) WHERE id(m)=="Tim Duncan" @@ -205,31 +203,31 @@ Feature: Prune Properties rule | "Tim Duncan" | "Aron Baynes" | "Spurs" | "Aron Baynes" | | "Tim Duncan" | "Aron Baynes" | "Spurs" | "Boris Diaw" | And the execution plan should be: - | id | name | dependencies | operator info | - | 19 | DataCollect | 20 | | - | 20 | TopN | 23 | | - | 23 | Project | 13 | | - | 13 | BiInnerJoin | 9, 30 | | - | 9 | BiInnerJoin | 28, 29 | | - | 28 | Project | 5 | | - | 5 | AppendVertices | 4 | { "props": "[{\"props\":[\"name\"],\"tagId\":9}]" } | - | 4 | Traverse | 2 | { "vertexProps": "[{\"props\":[\"name\"],\"tagId\":9}]", "edgeProps": "[{\"type\": -5, \"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"]}, {\"type\": 5, \"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"]}, {\"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"], \"type\": -3}, {\"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"], \"type\": 3}, {\"type\": -4, \"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"]}, {\"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"], \"type\": 4}]" } | - | 2 | Dedup | 1 | | - | 1 | PassThrough | 3 | | - | 3 | Start | | | - | 29 | Project | 8 | | - | 8 | AppendVertices | 7 | { "props": "[{\"tagId\":10,\"props\":[\"name\"]}]" } | - | 7 | Traverse | 6 | { "vertexProps": "[{\"tagId\":9,\"props\":[\"name\"]}]", "edgeProps": "[{\"type\": -5, \"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"]}, {\"type\": 5, \"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"]}, {\"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"], \"type\": -3}, {\"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"], \"type\": 3}, {\"type\": -4, \"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"]}, {\"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"], \"type\": 4}]" } | - | 6 | Argument | | | - | 31 | Start | | | - | 30 | Project | 12 | | - | 12 | AppendVertices | 11 | { "props": "[{\"props\":[\"name\"],\"tagId\":9}]" } | - | 11 | Traverse | 10 | { "vertexProps": "[{\"props\":[\"name\"],\"tagId\":10}]", "edgeProps": "[{\"type\": -5, \"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"]}, {\"type\": 5, \"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"]}, {\"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"], \"type\": -3}, {\"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"], \"type\": 3}, {\"type\": -4, \"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"]}, {\"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"], \"type\": 4}]" } | - | 10 | Argument | | | + | id | name | dependencies | operator info | + | 20 | TopN | 23 | | + | 23 | Project | 13 | | + | 13 | BiInnerJoin | 9, 30 | | + | 9 | BiInnerJoin | 28, 29 | | + | 28 | Project | 5 | | + | 5 | AppendVertices | 4 | { "props": "[{\"props\":[\"name\"],\"tagId\":9}]" } | + | 4 | Traverse | 2 | { "vertexProps": "[{\"props\":[\"name\"],\"tagId\":9}]", "edgeProps": "[{\"type\": -5, \"props\": [\"_dst\", \"_rank\", \"_type\"]}, {\"type\": 5, \"props\": [\"_dst\", \"_rank\", \"_type\"]}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": -3}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": 3}, {\"type\": -4, \"props\": [\"_dst\", \"_rank\", \"_type\"]}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": 4}]" } | + | 2 | Dedup | 1 | | + | 1 | PassThrough | 3 | | + | 3 | Start | | | + | 29 | Project | 8 | | + | 8 | AppendVertices | 7 | { "props": "[{\"tagId\":10,\"props\":[\"name\"]}]" } | + | 7 | Traverse | 6 | { "vertexProps": "[{\"tagId\":9,\"props\":[\"name\"]}]", "edgeProps": "[{\"type\": -5, \"props\": [\"_dst\", \"_rank\", \"_type\"]}, {\"type\": 5, \"props\": [\"_dst\", \"_rank\", \"_type\"]}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": -3}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": 3}, {\"type\": -4, \"props\": [\"_dst\", \"_rank\", \"_type\"]}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": 4}]" } | + | 6 | Argument | | | + | 31 | Start | | | + | 30 | Project | 12 | | + | 12 | AppendVertices | 11 | { "props": "[{\"props\":[\"name\"],\"tagId\":9}]" } | + | 11 | Traverse | 10 | { "vertexProps": "[{\"props\":[\"name\"],\"tagId\":10}]", "edgeProps": "[{\"type\": -5, \"props\": [\"_dst\", \"_rank\", \"_type\"]}, {\"type\": 5, \"props\": [\"_dst\", \"_rank\", \"_type\"]}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": -3}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": 3}, {\"type\": -4, \"props\": [\"_dst\", \"_rank\", \"_type\"]}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": 4}]" } | + | 10 | Argument | | | # The schema id is not fixed in standalone cluster, so we skip it @distonly Scenario: Multi Match + Given a graph with space named "nba" When profiling query: """ MATCH (m)-[]-(n) WHERE id(m)=="Tim Duncan" @@ -252,7 +250,6 @@ Feature: Prune Properties rule | "Tim Duncan" | "Boris Diaw" | "Tim Duncan" | And the execution plan should be: | id | name | dependencies | operator info | - | 16 | DataCollect | 17 | | | 17 | TopN | 13 | | | 13 | Project | 12 | | | 12 | BiInnerJoin | 19, 11 | | @@ -287,28 +284,27 @@ Feature: Prune Properties rule | "Tim Duncan" | "Aron Baynes" | "Spurs" | "Aron Baynes" | | "Tim Duncan" | "Aron Baynes" | "Spurs" | "Boris Diaw" | And the execution plan should be: - | id | name | dependencies | operator info | - | 20 | DataCollect | 21 | | - | 21 | TopN | 17 | | - | 17 | Project | 16 | | - | 16 | BiInnerJoin | 23, 14 | | - | 23 | Project | 5 | | - | 5 | AppendVertices | 4 | { "props": "[{\"props\":[\"name\"],\"tagId\":9}]" } | - | 4 | Traverse | 2 | { "vertexProps": "[{\"props\":[\"name\"],\"tagId\":9}]" } | - | 2 | Dedup | 1 | | - | 1 | PassThrough | 3 | | - | 3 | Start | | | - | 14 | BiInnerJoin | 33, 34 | | - | 33 | Project | 10 | | - | 10 | AppendVertices | 9 | { "props": "[{\"tagId\":10,\"props\":[\"name\"]}]" } | - | 9 | Traverse | 8 | { "vertexProps": "[{\"tagId\":9,\"props\":[\"name\"]}]" } | - | 8 | Argument | | | - | 35 | Start | | | - | 34 | Project | 13 | | - | 13 | AppendVertices | 12 | { "props": "[{\"tagId\":9,\"props\":[\"name\"]}]" } | - | 12 | Traverse | 11 | { "vertexProps": "[{\"tagId\":10,\"props\":[\"name\"]}]", "edgeProps": "[{\"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"], \"type\": -5}, {\"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"], \"type\": 5}, {\"type\": -3, \"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"]}, {\"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"], \"type\": 3}, {\"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"], \"type\": -4}, {\"props\": [\"_dst\", \"_rank\", \"_type\", \"_src\"], \"type\": 4}]" } | - | 11 | Argument | | | - | 36 | Start | | | + | id | name | dependencies | operator info | + | 21 | TopN | 17 | | + | 17 | Project | 16 | | + | 16 | BiInnerJoin | 23, 14 | | + | 23 | Project | 5 | | + | 5 | AppendVertices | 4 | { "props": "[{\"props\":[\"name\"],\"tagId\":9}]" } | + | 4 | Traverse | 2 | { "vertexProps": "[{\"props\":[\"name\"],\"tagId\":9}]" } | + | 2 | Dedup | 1 | | + | 1 | PassThrough | 3 | | + | 3 | Start | | | + | 14 | BiInnerJoin | 33, 34 | | + | 33 | Project | 10 | | + | 10 | AppendVertices | 9 | { "props": "[{\"tagId\":10,\"props\":[\"name\"]}]" } | + | 9 | Traverse | 8 | { "vertexProps": "[{\"tagId\":9,\"props\":[\"name\"]}]" } | + | 8 | Argument | | | + | 35 | Start | | | + | 34 | Project | 13 | | + | 13 | AppendVertices | 12 | { "props": "[{\"tagId\":9,\"props\":[\"name\"]}]" } | + | 12 | Traverse | 11 | { "vertexProps": "[{\"tagId\":10,\"props\":[\"name\"]}]", "edgeProps": "[{\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": -5}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": 5}, {\"type\": -3, \"props\": [\"_dst\", \"_rank\", \"_type\"]}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": 3}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": -4}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": 4}]" } | + | 11 | Argument | | | + | 36 | Start | | | When profiling query: """ MATCH (v:player{name:"Tony Parker"}) @@ -321,21 +317,22 @@ Feature: Prune Properties rule | "Tim Duncan" | | "Tim Duncan" | And the execution plan should be: - | id | name | dependencies | operator info | - | 10 | Project | 11 | | - | 11 | BiInnerJoin | 14, 9 | | - | 14 | Project | 3 | | - | 3 | AppendVertices | 12 | { "props": "[{\"props\":[\"name\", \"age\", \"_tag\"],\"tagId\":9}, {\"props\":[\"name\", \"speciality\", \"_tag\"],\"tagId\":8}, {\"props\":[\"name\", \"_tag\"],\"tagId\":10}]" } | - | 12 | IndexScan | 2 | | - | 2 | Start | | | - | 9 | Project | 8 | | - | 8 | AppendVertices | 7 | { "props": "[{\"props\":[\"name\", \"age\", \"_tag\"],\"tagId\":9}, {\"props\":[\"name\", \"speciality\", \"_tag\"],\"tagId\":8}, {\"props\":[\"name\", \"_tag\"],\"tagId\":10}]" } | - | 7 | Traverse | 6 | { "vertexProps": "[{\"props\":[\"name\", \"age\", \"_tag\"],\"tagId\":9}, {\"props\":[\"name\", \"speciality\", \"_tag\"],\"tagId\":8}, {\"props\":[\"name\", \"_tag\"],\"tagId\":10}]", "edgeProps": "[{\"type\": -5, \"props\": [\"_src\", \"_type\", \"_rank\", \"_dst\", \"start_year\", \"end_year\"]}, {\"props\": [\"_src\", \"_type\", \"_rank\", \"_dst\", \"likeness\"], \"type\": -3}, {\"props\": [\"_src\", \"_type\", \"_rank\", \"_dst\", \"start_year\", \"end_year\"], \"type\": -4}]" } | - | 6 | Argument | | | + | id | name | dependencies | operator info | + | 10 | Project | 11 | | + | 11 | BiInnerJoin | 14, 9 | | + | 14 | Project | 3 | | + | 3 | AppendVertices | 12 | { "props": "[{\"props\":[\"name\"],\"tagId\":9}]" } | + | 12 | IndexScan | 2 | | + | 2 | Start | | | + | 9 | Project | 8 | | + | 8 | AppendVertices | 7 | { "props": "[{\"props\":[\"name\"],\"tagId\":9}]" } | + | 7 | Traverse | 6 | { "vertexProps": "", "edgeProps": "[{\"type\": -5, \"props\": [\"_dst\", \"_rank\", \"_type\"]}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": -3}, {\"props\": [\"_dst\", \"_rank\", \"_type\"], \"type\": -4}]" } | + | 6 | Argument | | | # The schema id is not fixed in standalone cluster, so we skip it @distonly Scenario: Optional Match + Given a graph with space named "nba" When profiling query: """ MATCH (m)-[]-(n) WHERE id(m)=="Tim Duncan" @@ -356,7 +353,6 @@ Feature: Prune Properties rule | "Tim Duncan" | "Manu Ginobili" | NULL | And the execution plan should be: | id | name | dependencies | operator info | - | 16 | DataCollect | 17 | | | 17 | TopN | 13 | | | 13 | Project | 12 | | | 12 | BiLeftJoin | 19, 11 | | @@ -381,15 +377,313 @@ Feature: Prune Properties rule | scount | | 270 | And the execution plan should be: - | id | name | dependencies | operator info | - | 12 | Aggregate | 13 | | - | 13 | BiInnerJoin | 15, 11 | | - | 15 | Project | 4 | | - | 4 | Traverse | 3 | { "vertexProps": "" } | - | 3 | Traverse | 14 | { "vertexProps": "" } | - | 14 | IndexScan | 2 | | - | 2 | Start | | | - | 11 | Project | 10 | | - | 10 | AppendVertices | 9 | { "props": "[{\"props\":[\"name\", \"age\", \"_tag\"],\"tagId\":9}, {\"props\":[\"name\", \"speciality\", \"_tag\"],\"tagId\":8}, {\"props\":[\"name\", \"_tag\"],\"tagId\":10}]" } | - | 9 | Traverse | 8 | { "vertexProps": "" } | - | 8 | Argument | | | + | id | name | dependencies | operator info | + | 12 | Aggregate | 13 | | + | 13 | BiInnerJoin | 15, 11 | | + | 15 | Project | 4 | | + | 4 | Traverse | 3 | { "vertexProps": "" } | + | 3 | Traverse | 14 | { "vertexProps": "" } | + | 14 | IndexScan | 2 | | + | 2 | Start | | | + | 11 | Project | 10 | | + | 10 | AppendVertices | 9 | { "props": "[{\"props\":[\"_tag\"],\"tagId\":9}]" } | + | 9 | Traverse | 8 | { "vertexProps": "" } | + | 8 | Argument | | | + + @distonly + Scenario: return function + Given a graph with space named "nba" + When profiling query: + """ + MATCH (v1)-[e:like*1..5]->(v2) + WHERE id(v1) == "Tim Duncan" + RETURN count(v2.player.age) + """ + Then the result should be, in order: + | count(v2.player.age) | + | 24 | + And the execution plan should be: + | id | name | dependencies | operator info | + | 7 | Aggregate | 6 | | + | 6 | Project | 5 | | + | 5 | AppendVertices | 4 | { "props": "[{\"props\":[\"age\"],\"tagId\":9}]" } | + | 4 | Traverse | 2 | {"vertexProps": "", "edgeProps": "[{\"type\": 3, \"props\": [\"_dst\", \"_rank\", \"_type\"]}]" } | + | 2 | Dedup | 1 | | + | 1 | PassThrough | 3 | | + | 3 | Start | | | + When profiling query: + """ + MATCH (v1)-[e:like*1..5]->(v2) + WHERE id(v1) == "Tim Duncan" + RETURN count(v2) + """ + Then the result should be, in order: + | count(v2) | + | 24 | + And the execution plan should be: + | id | name | dependencies | operator info | + | 7 | Aggregate | 6 | | + | 6 | Project | 5 | | + | 5 | AppendVertices | 4 | { "props": "[{\"props\":[\"_tag\"],\"tagId\":9}]" } | + | 4 | Traverse | 2 | {"vertexProps": "", "edgeProps": "[{\"type\": 3, \"props\": [\"_dst\", \"_rank\", \"_type\"]}]" } | + | 2 | Dedup | 1 | | + | 1 | PassThrough | 3 | | + | 3 | Start | | | + When profiling query: + """ + MATCH p = (v1)-[e:like*1..5]->(v2) + WHERE id(v1) == "Tim Duncan" + RETURN length(p) LIMIT 1 + """ + Then the result should be, in order: + | length(p) | + | 1 | + And the execution plan should be: + | id | name | dependencies | operator info | + | 13 | Project | 11 | | + | 11 | Limit | 5 | | + | 5 | AppendVertices | 4 | { "props": "[{\"props\":[\"_tag\"],\"tagId\":9}]" } | + | 4 | Traverse | 2 | {"vertexProps": "", "edgeProps": "[{\"type\": 3, \"props\": [\"_dst\", \"_rank\", \"_type\"]}]" } | + | 2 | Dedup | 1 | | + | 1 | PassThrough | 3 | | + | 3 | Start | | | + When profiling query: + """ + MATCH p = (a:player)-[e:like*1..3]->(b:player{age:39}) + WHERE id(a) == 'Yao Ming' + WITH b, length(p) AS distance + MATCH (b)-[:serve]->(t:team) + RETURN b.player.name, distance + """ + Then the result should be, in order: + | b.player.name | distance | + | "Tracy McGrady" | 1 | + | "Tracy McGrady" | 3 | + | "Tracy McGrady" | 1 | + | "Tracy McGrady" | 3 | + | "Tracy McGrady" | 1 | + | "Tracy McGrady" | 3 | + | "Tracy McGrady" | 1 | + | "Tracy McGrady" | 3 | + And the execution plan should be: + | id | name | dependencies | operator info | + | 14 | Project | 13 | | + | 13 | BiInnerJoin | 15,12 | | + | 15 | Project | 17 | | + | 17 | AppendVertices | 16 | { "props": "[{\"props\":[\"name\",\"age\"],\"tagId\":9}]" } | + | 16 | Traverse | 2 | {"vertexProps": "", "edgeProps": "[{\"type\": 3, \"props\": [\"_dst\", \"_rank\", \"_type\"]}]" } | + | 2 | Dedup | 1 | | + | 1 | PassThrough | 3 | | + | 3 | Start | | | + | 12 | Project | 18 | | + | 18 | AppendVertices | 10 | { "props": "[{\"props\":[\"_tag\"],\"tagId\":10}]" } | + | 10 | Traverse | 8 | {"vertexProps": "[{\"props\":[\"name\",\"age\"],\"tagId\":9}]", "edgeProps": "[{\"type\": 4, \"props\": [\"_dst\", \"_rank\", \"_type\"]}]" } | + | 8 | Argument | | | + | 9 | Start | | | + + @distonly + Scenario: union match + Given a graph with space named "nba" + When profiling query: + """ + MATCH (v:player{name:'Tim Duncan'})-[:like]->(b) RETURN id(b) + UNION + MATCH (a:player{age:36})-[:like]-(b) RETURN id(b) + """ + Then the result should be, in any order: + | id(b) | + | "Tony Parker" | + | "Manu Ginobili" | + | "Marco Belinelli" | + | "Dejounte Murray" | + | "Tim Duncan" | + | "Boris Diaw" | + | "LaMarcus Aldridge" | + | "Steve Nash" | + And the execution plan should be: + | id | name | dependencies | operator info | + | 14 | Dedup | 13 | | + | 13 | Union | 18, 19 | | + | 18 | Project | 4 | | + | 4 | AppendVertices | 20 | { "props": "[{\"props\":[\"_tag\"],\"tagId\":9}]" } | + | 20 | Traverse | 16 | {"vertexProps": "", "edgeProps": "[{\"type\": 3, \"props\": [\"_dst\", \"_rank\", \"_type\"]}]" } | + | 16 | IndexScan | 2 | | + | 2 | Start | | | + | 19 | Project | 10 | | + | 10 | AppendVertices | 21 | { "props": "[{\"props\":[\"_tag\"],\"tagId\":9}]" } | + | 21 | Traverse | 17 | {"vertexProps": "", "edgeProps": "[{\"type\": -3, \"props\": [\"_dst\", \"_rank\", \"_type\"]}, {\"type\": 3, \"props\": [\"_dst\", \"_rank\", \"_type\"]}]" } | + | 17 | IndexScan | 8 | | + | 8 | Start | | | + + @distonly + Scenario: optional match + Given a graph with space named "nba" + When profiling query: + """ + MATCH (v:player)-[:like]-(:player)<-[:teammate]-(b:player)-[:serve]->(t:team) + WHERE id(v) == 'Tim Duncan' AND b.player.age > 20 + WITH v, count(b) AS countB, t + OPTIONAL MATCH (v)-[:like]-()<-[:like]-(oldB)-[:serve]->(t) + WHERE oldB.player.age > 10 + WITH v, countB, t, count(oldB) AS cb + RETURN t.team.name, sum(countB) + """ + Then the result should be, in any order: + | t.team.name | sum(countB) | + | "Spurs" | 11 | + | "Hornets" | 3 | + And the execution plan should be: + | id | name | dependencies | operator info | + | 21 | Aggregate | 20 | | + | 20 | Aggregate | 19 | | + | 19 | BiLeftJoin | 10, 25 | | + | 10 | Aggregate | 23 | | + | 23 | Project | 22 | | + | 22 | Filter | 29 | | + | 29 | AppendVertices | 28 | { "props": "[{\"props\":[\"name\", \"_tag\"],\"tagId\":10}]" } | + | 28 | Traverse | 27 | {"vertexProps": "[{\"props\":[\"age\"],\"tagId\":9}]", "edgeProps": "[{\"type\": 4, \"props\": [\"_dst\", \"_rank\", \"_type\"]}]" } | + | 27 | Traverse | 26 | {"vertexProps": "", "edgeProps": "[{\"type\": -5, \"props\": [\"_dst\", \"_rank\", \"_type\"]}]" } | + | 26 | Traverse | 2 | {"vertexProps": "", "edgeProps": "[{\"type\": -3, \"props\": [\"_dst\", \"_rank\", \"_type\"]}, {\"type\": 3, \"props\": [\"_dst\", \"_rank\", \"_type\"]}]" } | + | 2 | Dedup | 1 | | + | 1 | PassThrough | 3 | | + | 3 | Start | | | + | 25 | Project | 24 | | + | 24 | Filter | 16 | | + | 16 | AppendVertices | 15 | { "props": "[{\"props\":[\"name\", \"_tag\"],\"tagId\":10}]" } | + | 15 | Traverse | 14 | {"vertexProps": "[{\"props\":[\"age\"],\"tagId\":9}]", "edgeProps": "[{\"type\": 4, \"props\": [\"_dst\", \"_rank\", \"_type\"]}]" } | + | 14 | Traverse | 13 | {"vertexProps": "", "edgeProps": "[{\"type\": -3, \"props\": [\"_dst\", \"_rank\", \"_type\"]}]" } | + | 13 | Traverse | 11 | {"vertexProps": "", "edgeProps": "[{\"type\": -3, \"props\": [\"_dst\", \"_rank\", \"_type\"]}, {\"type\": 3, \"props\": [\"_dst\", \"_rank\", \"_type\"]}]" } | + | 11 | Argument | | | + | 12 | Start | | | + + @distonly + Scenario: test properties: + Given an empty graph + And load "nba" csv data to a new space + And having executed: + """ + ALTER TAG player ADD (sex string NOT NULL DEFAULT "男"); + ALTER EDGE like ADD (start_year int64 NOT NULL DEFAULT 2022); + ALTER EDGE serve ADD (degree int64 NOT NULL DEFAULT 88); + """ + And wait 6 seconds + When executing query: + """ + match (v) return properties(v).name limit 2; + """ + Then the result should be, in any order, with relax comparison: + | properties(v).name | + | "Amar'e Stoudemire" | + | "Carmelo Anthony" | + When executing query: + """ + match (v:player) return properties(v).name limit 2; + """ + Then the result should be, in any order, with relax comparison: + | properties(v).name | + | "Amar'e Stoudemire" | + | "Carmelo Anthony" | + When executing query: + """ + match (v:player) return properties(v).name,v.player.age limit 2; + """ + Then the result should be, in any order, with relax comparison: + | properties(v).name | v.player.age | + | "Amar'e Stoudemire" | 36 | + | "Carmelo Anthony" | 34 | + When executing query: + """ + match (v:player) where properties(v).name == "LaMarcus Aldridge" return properties(v).age; + """ + Then the result should be, in any order, with relax comparison: + | properties(v).age | + | 33 | + When executing query: + """ + match (v:player) where properties(v).name == "LaMarcus Aldridge" return v.player.age; + """ + Then the result should be, in any order, with relax comparison: + | v.player.age | + | 33 | + When executing query: + """ + match (v:player) where properties(v).name=="LaMarcus Aldridge" return v.player.sex,properties(v).age; + """ + Then the result should be, in any order, with relax comparison: + | v.player.sex | properties(v).age | + | "男" | 33 | + When executing query: + """ + match (v:player) where id(v)=="Carmelo Anthony" return v.player.age; + """ + Then the result should be, in any order, with relax comparison: + | v.player.age | + | 34 | + When executing query: + """ + match (v:player) where id(v)=="Carmelo Anthony" return properties(v).age; + """ + Then the result should be, in any order, with relax comparison: + | properties(v).age | + | 34 | + When executing query: + """ + match (v:player) where id(v)=="Carmelo Anthony" return properties(v).age,v.player.sex; + """ + Then the result should be, in any order, with relax comparison: + | properties(v).age | v.player.sex | + | 34 | "男" | + When executing query: + """ + match (v:player{name:"LaMarcus Aldridge"}) return v.player.age; + """ + Then the result should be, in any order, with relax comparison: + | v.player.age | + | 33 | + When executing query: + """ + match (v:player{name:"LaMarcus Aldridge"}) return properties(v).age; + """ + Then the result should be, in any order, with relax comparison: + | properties(v).age | + | 33 | + When executing query: + """ + match (v:player{name:"LaMarcus Aldridge"}) return properties(v).age,v.player.sex; + """ + Then the result should be, in any order, with relax comparison: + | properties(v).age | v.player.sex | + | 33 | "男" | + When executing query: + """ + match (v:player) return id(v),properties(v).name,v.player.age limit 2; + """ + Then the result should be, in any order, with relax comparison: + | id(v) | properties(v).name | v.player.age | + | "Amar'e Stoudemire" | "Amar'e Stoudemire" | 36 | + | "Carmelo Anthony" | "Carmelo Anthony" | 34 | + When executing query: + """ + match (v) return id(v),properties(v).name,v.player.age limit 2; + """ + Then the result should be, in any order, with relax comparison: + | id(v) | properties(v).name | v.player.age | + | "Amar'e Stoudemire" | "Amar'e Stoudemire" | 36 | + | "Carmelo Anthony" | "Carmelo Anthony" | 34 | + When executing query: + """ + match ()-[e]->() return properties(e).start_year limit 2; + """ + Then the result should be, in any order, with relax comparison: + | properties(e).start_year | + | 2022 | + | 2015 | + When executing query: + """ + match ()-[e:serve]->() where e.start_year>1022 return properties(e).degree limit 2; + """ + Then the result should be, in any order, with relax comparison: + | properties(e).degree | + | 88 | + | 88 | + Then drop the used space diff --git a/tests/tck/features/optimizer/PushFilterDownAggregateRule.feature b/tests/tck/features/optimizer/PushFilterDownAggregateRule.feature index 44d5c5fbfca..8d1b0a78784 100644 --- a/tests/tck/features/optimizer/PushFilterDownAggregateRule.feature +++ b/tests/tck/features/optimizer/PushFilterDownAggregateRule.feature @@ -36,7 +36,6 @@ Feature: Push Filter down Aggregate rule | 29 | 3 | And the execution plan should be: | id | name | dependencies | operator info | - | 13 | DataCollect | 12 | | | 12 | Sort | 19 | | | 19 | Aggregate | 18 | | | 18 | Filter | 8 | | diff --git a/tests/tck/features/optimizer/PushFilterDownProjectRule.feature b/tests/tck/features/optimizer/PushFilterDownProjectRule.feature index b1e6e7df325..818dceb53f6 100644 --- a/tests/tck/features/optimizer/PushFilterDownProjectRule.feature +++ b/tests/tck/features/optimizer/PushFilterDownProjectRule.feature @@ -61,7 +61,6 @@ Feature: Push Filter down Project rule | ("Tim Duncan" :bachelor{name: "Tim Duncan", speciality: "psychology"} :player{age: 42, name: "Tim Duncan"}) | ("Manu Ginobili" :player{age: 41, name: "Manu Ginobili"}) | 30 | And the execution plan should be: | id | name | dependencies | operator info | - | 25 | DataCollect | 24 | | | 24 | Dedup | 41 | | | 41 | Project | 40 | | | 40 | Filter | 20 | | diff --git a/tests/tck/features/optimizer/RemoveUselessProjectRule.feature b/tests/tck/features/optimizer/RemoveUselessProjectRule.feature index 7fda5985b8e..8d0da733d09 100644 --- a/tests/tck/features/optimizer/RemoveUselessProjectRule.feature +++ b/tests/tck/features/optimizer/RemoveUselessProjectRule.feature @@ -50,7 +50,6 @@ Feature: Remove Useless Project Rule | 48 | 1 | And the execution plan should be: | id | name | dependencies | operator info | - | 7 | DataCollect | 6 | | | 6 | Sort | 8 | | | 8 | Aggregate | 2 | | | 2 | AppendVertices | 1 | | diff --git a/tests/tck/features/optimizer/TopNRule.feature b/tests/tck/features/optimizer/TopNRule.feature index a4414d61f66..547a3ae637f 100644 --- a/tests/tck/features/optimizer/TopNRule.feature +++ b/tests/tck/features/optimizer/TopNRule.feature @@ -20,7 +20,6 @@ Feature: TopN rule | 55 | And the execution plan should be: | id | name | dependencies | operator info | - | 0 | DataCollect | 1 | | | 1 | TopN | 2 | | | 2 | Project | 3 | | | 3 | GetNeighbors | 4 | | @@ -39,7 +38,6 @@ Feature: TopN rule | 83 | And the execution plan should be: | id | name | dependencies | operator info | - | 0 | DataCollect | 1 | | | 1 | TopN | 2 | | | 2 | Project | 3 | | | 3 | GetNeighbors | 4 | | @@ -58,7 +56,6 @@ Feature: TopN rule | 60 | And the execution plan should be: | id | name | dependencies | operator info | - | 0 | DataCollect | 1 | | | 1 | Limit | 2 | | | 2 | Sort | 3 | | | 3 | Project | 4 | | @@ -79,7 +76,6 @@ Feature: TopN rule | 60 | And the execution plan should be: | id | name | dependencies | operator info | - | 0 | DataCollect | 1 | | | 1 | Sort | 2 | | | 2 | Project | 3 | | | 3 | GetNeighbors | 4 | |