Skip to content

Commit

Permalink
wip: early filter/drop unused feature properties
Browse files Browse the repository at this point in the history
  • Loading branch information
@hjanetzek
hjanetzek committed Dec 10, 2018
1 parent 014c3c5 commit 52ffd18
Show file tree
Hide file tree
Showing 5 changed files with 93 additions and 34 deletions.
8 changes: 8 additions & 0 deletions core/include/tangram/data/tileSource.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,12 @@ class TileSource : public std::enable_shared_from_this<TileSource> {

void setFormat(Format format) { m_format = format; }

struct PropertyFilter {
std::vector<std::string> drop;
std::vector<std::string> keep;
};
void setPropertyFilter(PropertyFilter&& filter) { m_propertyFilter = std::move(filter); }

protected:

void createSubTasks(std::shared_ptr<TileTask> _task);
Expand All @@ -167,6 +173,8 @@ class TileSource : public std::enable_shared_from_this<TileSource> {
std::vector<std::shared_ptr<TileSource>> m_rasterSources;

std::unique_ptr<DataSource> m_sources;

PropertyFilter m_propertyFilter;
};

}
38 changes: 32 additions & 6 deletions core/src/data/formats/mvt.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,12 +117,14 @@ Feature Mvt::getFeature(ParserContext& _ctx, protobuf::message _featureIn) {
}

auto valueKey = tagsMsg.varint();

// Check if the property should be dropped
if (_ctx.keys[tagKey].empty()) {
continue;
}
if( _ctx.values.size() <= valueKey ) {
LOGE("accessing out of bound values");
return feature;
}

_ctx.featureTags[tagKey] = valueKey;
}
break;
Expand Down Expand Up @@ -215,7 +217,8 @@ Feature Mvt::getFeature(ParserContext& _ctx, protobuf::message _featureIn) {
return feature;
}

Layer Mvt::getLayer(ParserContext& _ctx, protobuf::message _layerIn) {
Layer Mvt::getLayer(ParserContext& _ctx, protobuf::message _layerIn,
const TileSource::PropertyFilter& filter) {

Layer layer("");

Expand Down Expand Up @@ -245,7 +248,29 @@ Layer Mvt::getLayer(ParserContext& _ctx, protobuf::message _layerIn) {
continue;
}
case LAYER_KEY: {
_ctx.keys.push_back(_layerIn.string());
std::string key = _layerIn.string();
// Check whether the key must be kept
if (std::find(std::begin(filter.keep), std::end(filter.keep), key) == std::end(filter.keep)) {
// Check whether the key should be dropped
if (std::find_if(std::begin(filter.drop), std::end(filter.drop),
[&](auto& k) {
if (k.back() == '*' && key.length() >= k.length()-1) {
int n = std::strncmp(key.c_str(), k.c_str(), k.length()-1);
//LOG("check %s / %d / %d", k.c_str(), n, k.length()-1);
return n == 0;
} else {
return key == k;
}}) != std::end(filter.drop)) {

LOG("drop key: %s", key.c_str());
key = "";
//} else {
//LOG("keep key: %s", key.c_str());
}
//} else {
//LOG("keep key: %s", key.c_str());
}
_ctx.keys.emplace_back(std::move(key));
break;
}
case LAYER_VALUE: {
Expand Down Expand Up @@ -321,7 +346,8 @@ Layer Mvt::getLayer(ParserContext& _ctx, protobuf::message _layerIn) {
return layer;
}

std::shared_ptr<TileData> Mvt::parseTile(const TileTask& _task, int32_t _sourceId) {
std::shared_ptr<TileData> Mvt::parseTile(const TileTask& _task, int32_t _sourceId,
const TileSource::PropertyFilter& filter) {

auto tileData = std::make_shared<TileData>();

Expand All @@ -333,7 +359,7 @@ std::shared_ptr<TileData> Mvt::parseTile(const TileTask& _task, int32_t _sourceI
try {
while(item.next()) {
if(item.tag == LAYER) {
tileData->layers.push_back(getLayer(ctx, item.getMessage()));
tileData->layers.push_back(getLayer(ctx, item.getMessage(), filter));
} else {
item.skip();
}
Expand Down
57 changes: 30 additions & 27 deletions core/src/data/formats/mvt.h
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#pragma once

#include "data/tileData.h"
#include "data/tileSource.h"
#include "pbf/pbf.hpp"
#include "util/variant.h"

Expand All @@ -16,41 +17,43 @@ class MapProjection;

namespace Mvt {

struct Geometry {
std::vector<Point> coordinates;
std::vector<int> sizes;
};
struct Geometry {
std::vector<Point> coordinates;
std::vector<int> sizes;
};

struct ParserContext {
ParserContext(int32_t _sourceId) : sourceId(_sourceId){}
struct ParserContext {
explicit ParserContext(int32_t _sourceId) : sourceId(_sourceId){}

int32_t sourceId;
std::vector<std::string> keys;
std::vector<Value> values;
std::vector<protobuf::message> featureMsgs;
Geometry geometry;
// Map Key ID -> Tag values
std::vector<int> featureTags;
// Key IDs sorted by Property key ordering
std::vector<int> orderedKeys;
int32_t sourceId;
std::vector<std::string> keys;
std::vector<Value> values;
std::vector<protobuf::message> featureMsgs;
Geometry geometry;
// Map Key ID -> Tag values
std::vector<int> featureTags;
// Key IDs sorted by Property key ordering
std::vector<int> orderedKeys;

int tileExtent = 0;
int winding = 0;
};
int tileExtent = 0;
int winding = 0;
};

enum GeomCmd {
moveTo = 1,
lineTo = 2,
closePath = 7
};
enum GeomCmd {
moveTo = 1,
lineTo = 2,
closePath = 7
};

Geometry getGeometry(ParserContext& _ctx, protobuf::message _geomIn);
Geometry getGeometry(ParserContext& _ctx, protobuf::message _geomIn);

Feature getFeature(ParserContext& _ctx, protobuf::message _featureIn);
Feature getFeature(ParserContext& _ctx, protobuf::message _featureIn);

Layer getLayer(ParserContext& _ctx, protobuf::message _layerIn);
Layer getLayer(ParserContext& _ctx, protobuf::message _layerIn,
const TileSource::PropertyFilter& filter);

std::shared_ptr<TileData> parseTile(const TileTask& _task, int32_t _sourceId);
std::shared_ptr<TileData> parseTile(const TileTask& _task, int32_t _sourceId,
const TileSource::PropertyFilter& filter);

} // namespace Mvt

Expand Down
2 changes: 1 addition & 1 deletion core/src/data/tileSource.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ std::shared_ptr<TileData> TileSource::parse(const TileTask& _task) const {
switch (m_format) {
case Format::TopoJson: return TopoJson::parseTile(_task, m_id);
case Format::GeoJson: return GeoJson::parseTile(_task, m_id);
case Format::Mvt: return Mvt::parseTile(_task, m_id);
case Format::Mvt: return Mvt::parseTile(_task, m_id, m_propertyFilter);
}
assert(false);
return nullptr;
Expand Down
22 changes: 22 additions & 0 deletions core/src/scene/sceneLoader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -969,6 +969,26 @@ void SceneLoader::loadSource(const std::shared_ptr<Platform>& platform, const st
}
}

TileSource::PropertyFilter propFilter;
if (auto n = source["drop_feature_properties"]) {
if (n.IsSequence()) {
for (auto& key : n) {
if (key.IsScalar()) {
propFilter.drop.push_back(key.Scalar());
}
}
}
}
if (auto n = source["keep_feature_properties"]) {
if (n.IsSequence()) {
for (auto& key : n) {
if (key.IsScalar()) {
propFilter.keep.push_back(key.Scalar());
}
}
}
}

// Parse and append any URL parameters.
if (auto urlParamsNode = source["url_params"]) {
std::stringstream urlStream;
Expand Down Expand Up @@ -1084,6 +1104,8 @@ void SceneLoader::loadSource(const std::shared_ptr<Platform>& platform, const st
"This source will be ignored.", name.c_str());
return;
}

sourcePtr->setPropertyFilter(std::move(propFilter));
}

_scene->tileSources().push_back(sourcePtr);
Expand Down

0 comments on commit 52ffd18

Please sign in to comment.