diff --git a/.gitignore b/.gitignore index b1d25267..18805e51 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # object files created during the build and the output binary itself *.o tilemaker +tilemaker-server build/ tilemaker.dSYM/ diff --git a/CHANGELOG.md b/CHANGELOG.md index b692daaa..396ec898 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,46 @@ # Changelog +## [3.0.0] - 2024-01-15 + +3.0 is a major release that significantly reduces tilemaker's memory footprint and improves running time. Note that it has __breaking changes__ in the way you write your Lua scripts (`way:Layer` becomes simply `Layer`, and so on). + +### Added +- PMTiles output (@systemed) +- C++ tilemaker-server for quick prototyping (@bdon) +- GeoJSON supported as an alternative to shapefiles (@systemed) +- Support nodes in relations and relation roles (@cldellow) +- Nested relations support (@systemed/@cldellow) +- `LayerAsCentroid` can use positions from relation child nodes (@cldellow) +- Add polylabel algorithm to `LayerAsCentroid` (@cldellow) +- Filter input .pbf by way keys (@cldellow) +- GeoJSON writer for debugging (@systemed) +- Warn about PBFs with large blocks (@cldellow) +- Unit tests for various features (@cldellow) +- `RestartRelations()` to reset relation iterator (@systemed) +- Per-layer, zoom-dependent feature_limit (@systemed after an original by @keichan34) +- Report OSM ID on Lua processing error (@systemed) +- Docker OOM killer warning (@Firefishy) +- Push Docker image to Github package (@JinIgarashi) +- Support `type=boundary` relations as native multipolygons (@systemed) + +### Changed +- __BREAKING__: Lua calls use the global namespace, so `Layer` instead of `way:Layer` etc. (@cldellow) +- __BREAKING__: Mapsplit (.msf) support removed (@systemed) +- Widespread speed improvements (@cldellow, @systemed) +- Reduced memory consumption (@cldellow) +- protobuf dependency removed: protozero/vtzero used instead (@cldellow) +- Better Lua detection in Makefile (@systemed) +- z-order is now a lossy float: compile-time flag not needed (@systemed) +- --input and --output parameter names no longer required explicitly (@systemed) +- Docker image improvements (@Booligoosh) + +### Fixed +- Improved polygon correction (@systemed) +- Add missing attributes to OMT layers (@Nakaner) +- Use different OSM tags for OMT subclasses (@Nakaner) +- Add access and mtb_scale attributes to OMT (@dschep) +- Fix CMake build on Arch Linux (@holzgeist) + ## [2.4.0] - 2023-03-28 diff --git a/CMakeLists.txt b/CMakeLists.txt index f65bbc20..3ff4b32e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -96,8 +96,10 @@ file(GLOB tilemaker_src_files src/shared_data.cpp src/shp_mem_tiles.cpp src/shp_processor.cpp + src/significant_tags.cpp src/sorted_node_store.cpp src/sorted_way_store.cpp + src/tag_map.cpp src/tile_data.cpp src/tilemaker.cpp src/tile_worker.cpp diff --git a/Makefile b/Makefile index ba3f354a..8ee402bb 100644 --- a/Makefile +++ b/Makefile @@ -1,82 +1,71 @@ # See what Lua versions are installed -# order of preference: LuaJIT 2.1, LuaJIT 2.0, any generic Lua, Lua 5.1 +# order of preference: LuaJIT, any generic Lua, then versions from 5.4 down PLATFORM_PATH := /usr/local -ifneq ("$(wildcard /usr/local/include/luajit-2.1/lua.h)","") - LUA_VER := LuaJIT 2.1 - LUA_CFLAGS := -I/usr/local/include/luajit-2.1 -DLUAJIT - LUA_LIBS := -lluajit-5.1 - LUAJIT := 1 - -else ifneq ("$(wildcard /usr/include/luajit-2.1/lua.h)","") - LUA_VER := LuaJIT 2.1 - LUA_CFLAGS := -I/usr/include/luajit-2.1 -DLUAJIT - LUA_LIBS := -lluajit-5.1 - LUAJIT := 1 - -else ifneq ("$(wildcard /usr/local/include/luajit-2.0/lua.h)","") - LUA_VER := LuaJIT 2.0 - LUA_CFLAGS := -I/usr/local/include/luajit-2.0 -DLUAJIT - LUA_LIBS := -lluajit-5.1 - LUAJIT := 1 - -else ifneq ("$(wildcard /usr/include/luajit-2.0/lua.h)","") - LUA_VER := LuaJIT 2.0 - LUA_CFLAGS := -I/usr/include/luajit-2.0 -DLUAJIT - LUA_LIBS := -lluajit-5.1 - LUAJIT := 1 - -else ifneq ("$(wildcard /usr/local/include/lua/lua.h)","") - LUA_VER := system Lua - LUA_CFLAGS := -I/usr/local/include/lua - LUA_LIBS := -llua - -else ifneq ("$(wildcard /usr/include/lua/lua.h)","") - LUA_VER := system Lua - LUA_CFLAGS := -I/usr/include/lua - LUA_LIBS := -llua - -else ifneq ("$(wildcard /usr/include/lua.h)","") - LUA_VER := system Lua +# First, find what the Lua executable is called +# - when a new Lua is released, then add it before 5.4 here +LUA_CMD := $(shell luajit -e 'print("luajit")' 2> /dev/null || lua -e 'print("lua")' 2> /dev/null || lua5.4 -e 'print("lua5.4")' 2> /dev/null || lua5.3 -e 'print("lua5.3")' 2> /dev/null || lua5.2 -e 'print("lua5.2")' 2> /dev/null || lua5.1 -e 'print("lua5.1")' 2> /dev/null) +ifeq ($(LUA_CMD),"") + $(error Couldn't find Lua interpreter) +endif +$(info Using ${LUA_CMD}) + +# Find the language version +LUA_LANGV := $(shell ${LUA_CMD} -e 'print(string.match(_VERSION, "%d+.%d+"))') +$(info - Lua language version ${LUA_LANGV}) + +# Find the directory where Lua might be +ifeq ($(LUA_CMD),luajit) + # We need the LuaJIT version (2.0/2.1) to find this + LUA_JITV := $(shell luajit -e 'a,b,c=string.find(jit.version,"LuaJIT (%d.%d)");print(c)') + $(info - LuaJIT version ${LUA_JITV}) + LUA_DIR := luajit-${LUA_JITV} + LUA_LIBS := -lluajit-${LUA_LANGV} +else + LUA_DIR := $(LUA_CMD) + LUA_LIBS := -l${LUA_CMD} +endif + +# Find the include path by looking in the most likely locations +ifneq ('$(wildcard /usr/local/include/${LUA_DIR}/lua.h)','') + LUA_CFLAGS := -I/usr/local/include/${LUA_DIR} +else ifneq ('$(wildcard /usr/local/include/${LUA_DIR}${LUA_LANGV}/lua.h)','') + LUA_CFLAGS := -I/usr/local/include/${LUA_DIR}${LUA_LANGV} + LUA_LIBS := -l${LUA_CMD}${LUA_LANGV} +else ifneq ('$(wildcard /usr/include/${LUA_DIR}/lua.h)','') + LUA_CFLAGS := -I/usr/include/${LUA_DIR} +else ifneq ('$(wildcard /usr/include/${LUA_DIR}${LUA_LANGV}/lua.h)','') + LUA_CFLAGS := -I/usr/include/${LUA_DIR}${LUA_LANGV} + LUA_LIBS := -l${LUA_CMD}${LUA_LANGV} +else ifneq ('$(wildcard /usr/include/lua.h)','') LUA_CFLAGS := -I/usr/include - LUA_LIBS := -llua - -else ifneq ("$(wildcard /usr/local/include/lua5.1/lua.h)","") - LUA_VER := Lua 5.1 - LUA_CFLAGS := -I/usr/local/include/lua5.1 - LUA_LIBS := -llua5.1 - -else ifneq ("$(wildcard /usr/include/lua5.1/lua.h)","") - LUA_VER := Lua 5.1 - LUA_CFLAGS := -I/usr/include/lua5.1 - LUA_LIBS := -llua5.1 - -else ifneq ("$(wildcard /usr/include/lua5.3/lua.h)","") - LUA_VER := Lua 5.3 - LUA_CFLAGS := -I/usr/include/lua5.3 - LUA_LIBS := -llua5.3 - -else ifneq ("$(wildcard /opt/homebrew/include/lua5.1/lua.h)","") - LUA_VER := Lua 5.1 - LUA_CFLAGS := -I/opt/homebrew/include/lua5.1 - LUA_LIBS := -llua5.1 +else ifneq ('$(wildcard /opt/homebrew/include/${LUA_DIR}/lua.h)','') + LUA_CFLAGS := -I/opt/homebrew/include/${LUA_DIR} + PLATFORM_PATH := /opt/homebrew +else ifneq ('$(wildcard /opt/homebrew/include/${LUA_DIR}${LUA_LANGV}/lua.h)','') + LUA_CFLAGS := -I/opt/homebrew/include/${LUA_DIR}${LUA_LANGV} + LUA_LIBS := -l${LUA_CMD}${LUA_LANGV} PLATFORM_PATH := /opt/homebrew - else - $(error Couldn't find Lua) + $(error Couldn't find Lua libraries) endif -$(info Using ${LUA_VER} (include path is ${LUA_CFLAGS}, library path is ${LUA_LIBS})) -ifneq ($(OS),Windows_NT) - ifeq ($(shell uname -s), Darwin) - ifeq ($(LUAJIT), 1) +# Append LuaJIT-specific flags if needed +ifeq ($(LUA_CMD),luajit) + LUA_CFLAGS := ${LUA_CFLAGS} -DLUAJIT + ifneq ($(OS),Windows_NT) + ifeq ($(shell uname -s), Darwin) LDFLAGS := -pagezero_size 10000 -image_base 100000000 $(info - with MacOS LuaJIT linking) endif endif endif +# Report success +$(info - include path is ${LUA_CFLAGS}) +$(info - library path is ${LUA_LIBS}) + # Main includes prefix = /usr/local @@ -121,8 +110,10 @@ tilemaker: \ src/shared_data.o \ src/shp_mem_tiles.o \ src/shp_processor.o \ + src/significant_tags.o \ src/sorted_node_store.o \ src/sorted_way_store.o \ + src/tag_map.o \ src/tile_data.o \ src/tilemaker.o \ src/tile_worker.o \ @@ -136,6 +127,7 @@ test: \ test_pbf_reader \ test_pooled_string \ test_relation_roles \ + test_significant_tags \ test_sorted_node_store \ test_sorted_way_store @@ -166,11 +158,18 @@ test_pooled_string: \ test/pooled_string.test.o $(CXX) $(CXXFLAGS) -o test.pooled_string $^ $(INC) $(LIB) $(LDFLAGS) && ./test.pooled_string + test_relation_roles: \ src/relation_roles.o \ test/relation_roles.test.o $(CXX) $(CXXFLAGS) -o test.relation_roles $^ $(INC) $(LIB) $(LDFLAGS) && ./test.relation_roles +test_significant_tags: \ + src/significant_tags.o \ + src/tag_map.o \ + test/significant_tags.test.o + $(CXX) $(CXXFLAGS) -o test.significant_tags $^ $(INC) $(LIB) $(LDFLAGS) && ./test.significant_tags + test_sorted_node_store: \ src/external/streamvbyte_decode.o \ src/external/streamvbyte_encode.o \ @@ -209,8 +208,8 @@ install: install -m 0755 -d $(DESTDIR)$(prefix)/bin/ install -m 0755 tilemaker $(DESTDIR)$(prefix)/bin/ install -m 0755 tilemaker-server $(DESTDIR)$(prefix)/bin/ - install -m 0755 -d ${DESTDIR}${MANPREFIX}/man1/ - install docs/man/tilemaker.1 ${DESTDIR}${MANPREFIX}/man1/ + @install -m 0755 -d ${DESTDIR}${MANPREFIX}/man1/ || true + @install docs/man/tilemaker.1 ${DESTDIR}${MANPREFIX}/man1/ || true clean: rm -f tilemaker tilemaker-server src/*.o src/external/*.o include/*.o include/*.pb.h server/*.o test/*.o diff --git a/README.md b/README.md index 05345723..3672b693 100644 --- a/README.md +++ b/README.md @@ -12,14 +12,13 @@ See an example of a vector tile map produced by tilemaker at [tilemaker.org](htt tilemaker is written in C++14. The chief dependencies are: -* Google Protocol Buffers * Boost (latest version advised, 1.66 minimum) * Lua (5.1 or later) or LuaJIT * sqlite3 * shapelib * rapidjson -sqlite_modern_cpp, and kaguya are bundled in the include/ directory. +Other third-party code is bundled in the include/ directory. You can then simply install with: @@ -32,20 +31,19 @@ For detailed installation instructions for your operating system, see [INSTALL.m tilemaker comes with configuration files compatible with the popular [OpenMapTiles](https://openmaptiles.org) schema, and a demonstration map server. You'll run tilemaker to make vector tiles from your `.osm.pbf` source data. To create the tiles, run this from the tilemaker directory: - tilemaker --input /path/to/your/input.osm.pbf \ - --output /path/to/your/output.mbtiles + tilemaker /path/to/your/input.osm.pbf /path/to/your/output.mbtiles If you want to include sea tiles, then create a directory called `coastline` in the same place you're running tilemaker from, and then save the files from https://osmdata.openstreetmap.de/download/water-polygons-split-4326.zip in it, such that tilemaker can find a file at `coastline/water_polygons.shp`. +_(If you want to include optional small-scale landcover, create a `landcover` directory, and download the appropriate 10m files from 'Features' at https://www.naturalearthdata.com so that you have `landcover/ne_10m_antarctic_ice_shelves_polys/ne_10m_antarctic_ice_shelves_polys.shp`, `landcover/ne_10m_urban_areas/ne_10m_urban_areas.shp`, `landcover/ne_10m_glaciated_areas/ne_10m_glaciated_areas.shp`.)_ + Then, to serve your tiles using the demonstration server: cd server - ruby server.rb /path/to/your/output.mbtiles + tilemaker-server /path/to/your/output.mbtiles You can now navigate to http://localhost:8080/ and see your map! -(If you don't already have them, you'll need to install Ruby and the required gems to run the demonstration server. On Ubuntu, for example, `sudo apt install sqlite3 libsqlite3-dev ruby ruby-dev` and then `sudo gem install sqlite3 cgi glug rack rackup`.) - ## Your own configuration Vector tiles contain (generally thematic) 'layers'. For example, your tiles might contain river, cycleway and railway layers. It's up to you what OSM data goes into each layer. You configure this in tilemaker with two files: @@ -77,12 +75,11 @@ You might use tilemaker if: But don't use tilemaker if: * You want someone else to create and host the tiles for you -* You want the entire planet * You want continuous updates with the latest OSM data ## Contributing -Bug reports, suggestions and (especially!) pull requests are very welcome on the Github issue tracker. Please check the tracker to see if your issue is already known, and be nice. For questions, please use IRC (irc.oftc.net or https://irc.osm.org, channel #osm-dev) and https://help.osm.org. +Bug reports, suggestions and (especially!) pull requests are very welcome on the Github issue tracker. Please check the tracker to see if your issue is already known, and be nice. For questions, please use IRC (irc.oftc.net or https://irc.osm.org, channel #osm-dev) and https://community.osm.org. Formatting: braces and indents as shown, hard tabs (4sp). (Yes, I know.) Please be conservative about adding dependencies or increasing the memory requirement. @@ -90,14 +87,19 @@ Formatting: braces and indents as shown, hard tabs (4sp). (Yes, I know.) Please tilemaker is maintained by Richard Fairhurst and supported by [many contributors](https://github.com/systemed/tilemaker/graphs/contributors). -Copyright tilemaker contributors, 2015-2023. +Copyright tilemaker contributors, 2015-2024. The tilemaker code is licensed as FTWPL; you may do anything you like with this code and there is no warranty. Licenses of third-party libraries: -- sqlite_modern_cpp (Amin Roosta) is licensed under MIT +- [sqlite_modern_cpp](https://github.com/SqliteModernCpp/sqlite_modern_cpp) is licensed under MIT - [kaguya](https://github.com/satoren/kaguya) is licensed under the Boost Software Licence - [libpopcnt](https://github.com/kimwalisch/libpopcnt) is licensed under BSD 2-clause - [pmtiles](https://github.com/protomaps/PMTiles) is licensed under BSD 3-clause - [streamvbyte](https://github.com/lemire/streamvbyte) is licensed under Apache 2 +- [polylabel](https://github.com/mapbox/polylabel) is licensed under ISC +- [protozero](https://github.com/mapbox/protozero) is licensed under BSD 2-clause +- [vtzero](https://github.com/mapbox/vtzero) is licensed under BSD 2-clause +- [minunit](https://github.com/siu/minunit) is licensed under MIT +- [Simple-Web-Server](https://gitlab.com/eidheim/Simple-Web-Server) is licensed under MIT diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index d41fba9b..e4e2e6eb 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -2,11 +2,11 @@ Vector tiles contain (generally thematic) 'layers'. For example, your tiles might contain river, cycleway and railway layers. -You'll generally assign OpenStreetMap data into layers by making decisions based on their tags. You might put anything with a `highway=` tag into the roads layer, anything with a `railway=` tag into the railway layer, and so on. +You'll assign OpenStreetMap data into layers by making decisions based on their tags. You might put anything with a `highway=` tag into the roads layer, anything with a `railway=` tag into the railway layer, and so on. In tilemaker, you achieve this by writing a short script in the Lua programming language. Lua is a simple and fast language used by several other OpenStreetMap tools, such as the OSRM routing engine and osm2pgsql. -In addition, you supply tilemaker with a JSON file which specifies certain global settings for your tileset. +In addition, you supply tilemaker with a JSON file which lists the layers, and specifies certain global settings for your tileset. ### A note on zoom levels @@ -14,7 +14,15 @@ Because vector tiles are so efficiently encoded, you generally don't need to cre So when you set a maximum zoom level of 14 in tilemaker, this doesn't mean you're restricted to displaying maps at z14. It just means that tilemaker will create z14 tiles, and it's your renderer's job to use these tiles to draw the most detailed maps. -### JSON configuration +### Learning by doing + +This file is a complete reference to tilemaker's configuration. Impatient and want to get your hands dirty straightaway? Look in the `resources/` directory for config-example.json and process-example.lua. This is a really simple config example. Experiment with changing the layer descriptions in the JSON, and the feature logic in the Lua script. + +Make sure you've also read VECTOR_TILES.md (in `docs/`). This explains how tilemaker (which just generates the tiles) works together with a rendering library (like Maplibre) and stylesheet. + +## JSON configuration reference + +### Basic configuration The JSON config file sets out the layers you'll be using, and which zoom levels they apply to. For example, you might want to include your roads layer in your z12-z14 tiles, but your buildings at z14 only. @@ -54,8 +62,7 @@ A typical config file would look like this: The order of layers will be carried forward into the vector tile. -Layers with `write_to` set must appear after the layers they're writing into. -An incorrect order will result in "the layer to write doesn't exist". +Layers with `write_to` set must appear after the layers they're writing into. An incorrect order will result in "the layer to write doesn't exist". All options are compulsory unless stated otherwise. If tilemaker baulks at the JSON file, check everything's included, and run it through an online JSON validator to check for syntax errors. @@ -65,17 +72,19 @@ By default tilemaker expects to find this file at config.json, but you can speci You can add optional parameters to layers: -* `write_to` - write way/nodes to a previously named layer -* `simplify_below` - simplify ways below this zoom level -* `simplify_level` - how much to simplify ways (in degrees of longitude) on the zoom level `simplify_below-1` -* `simplify_length` - how much to simplify ways (in kilometers) on the zoom level `simplify_below-1`, preceding `simplify_level` +* `write_to` - write features to a previously named layer +* `simplify_below` - simplify features below this zoom level +* `simplify_level` - how much to simplify features (in degrees of longitude) on the zoom level `simplify_below-1` +* `simplify_length` - how much to simplify features (in kilometers) on the zoom level `simplify_below-1`, preceding `simplify_level` * `simplify_ratio` - (optional: the default value is 2.0) the actual simplify level will be `simplify_level * pow(simplify_ratio, (simplify_below-1) - )` * `filter_below` - filter areas by minimum size below this zoom level * `filter_area` - minimum size (in square degrees of longitude) for the zoom level `filter_below-1` +* `feature_limit` - restrict the number of features written to each tile +* `feature_limit_below` - restrict only below this zoom level * `combine_polygons_below` - merge adjacent polygons with the same attributes below this zoom level * `z_order_ascending` - sort features in ascending order by a numeric value set in the Lua processing script (defaults to `true`: specify `false` for descending order) -Use these options to combine different layer specs within one outputted layer. For example: +`write_to` enables you to combine different layer specs within one outputted layer. For example: "layers": { "roads": { "minzoom": 12, "maxzoom": 14 }, @@ -84,7 +93,7 @@ Use these options to combine different layer specs within one outputted layer. F This would combine the `roads` (z12-14) and `low_roads` (z9-11) layers into a single `roads` layer on writing, with simplified geometries for `low_roads`. -(See also 'Shapefiles' below.) +(See also 'Shapefiles and GeoJSON' below.) ### Additional metadata @@ -105,19 +114,23 @@ For example: } } -### Lua processing +## Lua processing reference -Your Lua file needs to supply 5 things: +Your Lua file can supply these functions for tilemaker to call: -1. `node_keys`, a list of those OSM keys which indicate that a node should be processed -2. `init_function(name)` (optional), a function to initialize Lua logic -2. `node_function(node)`, a function to process an OSM node and add it to layers -3. `way_function(way)`, a function to process an OSM way and add it to layers -3. `exit_function` (optional), a function to finalize Lua logic (useful to show statistics) +1. (optional) `node_keys`, a list of those OSM tags which indicate that a node should be processed +2. (optional) `way_keys`, a list of those OSM tags which indicate that a way should be processed +3. `node_function()`, a function to process an OSM node and add it to layers +4. `way_function()`, a function to process an OSM way and add it to layers +5. (optional) `init_function(name)`, a function to initialize Lua logic +6. (optional) `exit_function`, a function to finalize Lua logic (useful to show statistics) +7. (optional) `relation_scan_function`, a function to determine whether your Lua file wishes to process the given relation +8. (optional) `relation_function`, a function to process an OSM relation and add it to layers +9. (optional) `attribute_function`, a function to remap attributes from shapefiles -`node_keys` is a simple list (or in Lua parlance, a 'table') of OSM tag keys. If a node has one of those keys, it will be processed by `node_function`; if not, it'll be skipped. For example, if you wanted to show highway crossings and railway stations, it should be `{ "highway", "railway" }`. (This avoids the need to process the vast majority of nodes which contain no important tags at all.) +### Principal Lua functions -`node_function` and `way_function` work the same way. They are called with an OSM object; you then inspect the tags of that object, and put it in your vector tiles' layers based on those tags. In essence, the process is: +`node_function` and `way_function` do the main work. They are called with an OSM object; you then inspect the tags of that object, and put it in your vector tiles' layers based on those tags. In essence, the process is: * look at tags * if tags meet criteria, write to a layer @@ -127,44 +140,59 @@ Note the order: you write to a layer first, then set attributes after. To do that, you use these methods: -* `node:Find(key)` or `way:Find(key)`: get the value for a tag, or the empty string if not present. For example, `way:Find("railway")` might return "rail" for a railway, "siding" for a siding, or "" if it isn't a railway at all. -* `node:Holds(key)` or `way:Holds(key)`: returns true if that key exists, false otherwise. -* `node:Layer("layer_name", false)` or `way:Layer("layer_name", is_area)`: write this node/way to the named layer. This is how you put objects in your vector tile. is_area (true/false) specifies whether a way should be treated as an area, or just as a linestring. -* `way:LayerAsCentroid("layer_name")`: write a single centroid point for this way to the named layer (useful for labels and POIs). -* `node:Attribute(key,value,minzoom)` or `node:Attribute(key,value,minzoom)`: add an attribute to the most recently written layer. Argument `minzoom` is optional, use it if you do not want to write the attribute on lower zoom levels. -* `node:AttributeNumeric(key,value,minzoom)`, `node:AttributeBoolean(key,value,minzoom)` (and `way:`...): for numeric/boolean columns. -* `node:Id()` or `way:Id()`: get the OSM ID of the current object. -* `node:ZOrder(number)` or `way:ZOrder(number)`: Set a numeric value (default 0, 1-byte signed integer) used to sort features within a layer. Use this feature to ensure a proper rendering order if the rendering engine itself does not support sorting. Sorting is not supported across layers merged with `write_to`. Features with different z-order are not merged if `combine_below` or `combine_polygons_below` is used. -* `node:MinZoom(zoom)` or `way:MinZoom(zoom)`: set the minimum zoom level (0-15) at which this object will be written. Note that the JSON layer configuration minimum still applies (so `:MinZoom(5)` will have no effect if your layer only starts at z6). -* `way:Length()` and `way:Area()`: return the length (metres)/area (square metres) of the current object. Requires recent Boost. -* `way:Centroid()`: return the lat/lon of the centre of the current object as a two-element Lua table (element 1 is lat, 2 is lon). +* `Find(key)`: get the value for a tag, or the empty string if not present. For example, `Find("railway")` might return "rail" for a railway, "siding" for a siding, or "" if it isn't a railway at all. +* `Holds(key)`: returns true if that key exists, false otherwise. +* `Layer(layer_name, is_area)`: write this node/way to the named layer. This is how you put objects in your vector tile. is_area (true/false) specifies whether a way should be treated as an area, or just as a linestring. +* `LayerAsCentroid(layer_name, algorithm, role, role...)`: write a single centroid point for this way to the named layer (useful for labels and POIs). Only the first argument is required. `algorithm` can be "polylabel" (default) or "centroid". The third arguments onwards specify relation roles: if you're processing a multipolygon-type relation (e.g. a boundary) and it has a "label" node member, then by adding "label" as an argument here, this will be used in preference to the calculated point. +* `Attribute(key,value,minzoom)`: add an attribute to the most recently written layer. Argument `minzoom` is optional, use it if you do not want to write the attribute on lower zoom levels. +* `AttributeNumeric(key,value,minzoom)`, `AttributeBoolean(key,value,minzoom)`: for numeric/boolean columns. +* `Id()`: get the OSM ID of the current object. +* `ZOrder(number)`: Set a numeric value (default 0) used to sort features within a layer. Use this feature to ensure a proper rendering order if the rendering engine itself does not support sorting. Sorting is not supported across layers merged with `write_to`. Features with different z-order are not merged if `combine_below` or `combine_polygons_below` is used. Use this in conjunction with `feature_limit` to only write the most important (highest z-order) features within a tile. (Values can be -50,000,000 to 50,000,000 and are lossy, particularly beyond -1000 to 1000.) +* `MinZoom(zoom)`: set the minimum zoom level (0-15) at which this object will be written. Note that the JSON layer configuration minimum still applies (so `:MinZoom(5)` will have no effect if your layer only starts at z6). +* `Length()` and `Area()`: return the length (metres)/area (square metres) of the current object. Requires Boost 1.67+. +* `Centroid()`: return the lat/lon of the centre of the current object as a two-element Lua table (element 1 is lat, 2 is lon). The simplest possible function, to include roads/paths and nothing else, might look like this: - function way_function(way) - local highway = way:Find("highway") +```lua + function way_function() + local highway = Find("highway") if highway~="" then - way:Layer("roads", false) - way:Attribute("name", way:Find("name")) - way:Attribute("type", highway) + Layer("roads", false) + Attribute("name", Find("name")) + Attribute("type", highway) end end +``` Take a look at the supplied process.lua for a simple example, or the more complex OpenMapTiles-compatible script in `resources/`. You can specify another filename with the `--process` option. If your Lua file causes an error due to mistaken syntax, you can test it at the command line with `luac -p filename`. Three frequent Lua gotchas: tables (arrays) start at 1, not 0; the "not equal" operator is `~=` (that's the other way round from Perl/Ruby's regex operator); and `if` statements always need a `then`, even when written over several lines. +> [!IMPORTANT] +> If you are upgrading from a version of tilemaker before 3.0, note that you now write methods simply as `Find("key")` or `Layer("name")` etc., not as `way:Find("key")` or `way:Layer("name")`. + +### Other Lua functions and lists + +`node_keys` is a simple list (or in Lua parlance, a 'table') of OSM tags. If a node has one of those keys, it will be processed by `node_function`; if not, it'll be skipped. For example, if you wanted to show highway crossings and railway stations, it should be `{ "highway", "railway" }`. (This avoids the need to process the vast majority of nodes which contain no important tags at all.) + +`way_keys` is similar, but for ways. For ways, you may also wish to express the filter in terms of the tag value, or as an inversion. For example, to exclude buildings: `way_keys = {"~building"}`. To build a map only of major roads: `way_keys = {"highway=motorway", "highway=trunk", "highway=primary", "highway=secondary"}` + +`init_function(name)` and `exit_function` are called at the start and end of processing (once per thread). You can use this to output statistics or even to read a small amount of external data. + +Other functions are described below and in RELATIONS.md. + ### Relations Tilemaker handles multipolygon relations natively. The combined geometries are processed as ways (i.e. by `way_function`), so if your function puts buildings in a 'buildings' layer, tilemaker will cope with this whether the building is mapped as a simple way or a multipolygon. The only difference is that they're given an artificial ID. Multipolygons are expected to have tags on the relation, not the outer way. Working with other types of relations (e.g. routes) is documented in RELATIONS.md. -### Shapefiles +## Shapefiles and GeoJSON -Tilemaker chiefly works with OpenStreetMap .osm.pbf data, but you can also bring in shapefiles. These are useful for rarely-changing data such as coastlines and built-up area outlines. +Tilemaker chiefly works with OpenStreetMap .osm.pbf data, but you can also bring in external data in shapefile or GeoJSON format. This is useful for rarely-changing data such as coastlines and built-up area outlines. -Shapefiles are imported directly in your layer config like this: +Shapefiles and GeoJSON are imported directly in your layer config like this: "urban_areas": { "minzoom": 11, "maxzoom": 14, @@ -179,15 +207,15 @@ Shapefiles are imported directly in your layer config like this: You can specify attribute columns to import using the `source_columns` parameter, and they'll be available within your vector tiles just as any OSM tags that you import would be. To import all columns, use `"source_columns": true`. -Limited Lua transformations are available for shapefiles. You can supply an `attribute_function(attr,layer)` which takes a Lua table (hash) of shapefile attributes, as already filtered by `source_columns`, and the layer name. It must return a table (hash) of the vector tile attributes to set. +Limited Lua transformations are available for these files. You can supply an `attribute_function(attr,layer)` which takes a Lua table (hash) of shapefile attributes, as already filtered by `source_columns`, and the layer name. It must return a table (hash) of the vector tile attributes to set. To set the minimum zoom level at which an individual feature is rendered, use `attribute_function` to set a `_minzoom` value in your return table. -Shapefiles **must** be in WGS84 projection, i.e. pure latitude/longitude. (Use ogr2ogr to reproject them if your source material is in a different projection.) They will be clipped to the bounds of the first .pbf that you import, unless you specify otherwise with a `bounding_box` setting in your JSON file. +Shapefiles/GeoJSON **must** be in WGS84 projection, i.e. pure latitude/longitude. (Use ogr2ogr to reproject them if your source material is in a different projection.) They will be clipped to the bounds of the first .pbf that you import, unless you specify otherwise with a `bounding_box` setting in your JSON file. ### Lua spatial queries -When processing OSM objects with your Lua script, you can perform simple spatial queries against a shapefile layer. Let's say you have the following shapefile layer containing country polygons, each one named with the country name: +When processing OSM objects with your Lua script, you can perform simple spatial queries against a shapefile/GeoJSON layer. Let's say you have the following shapefile layer containing country polygons, each one named with the country name: "countries": { "minzoom": 2, "maxzoom": 10, @@ -197,11 +225,11 @@ When processing OSM objects with your Lua script, you can perform simple spatial You can then find out whether a node is within one of these polygons using the `Intersects` method: - if node:Intersects("countries") then print("Looks like it's on land"); end + if Intersects("countries") then print("Looks like it's on land"); end Or you can find out what country(/ies) the node is within using `FindIntersecting`, which returns a table: - names = node:FindIntersecting("countries") + names = FindIntersecting("countries") print(table.concat(name,",")) To enable these functions, set `index` to true in your shapefile layer definition. `index_column` is not needed for `Intersects` but required for `FindIntersecting`. diff --git a/docs/INSTALL.md b/docs/INSTALL.md index cdd88e75..e2ca6633 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -4,13 +4,15 @@ Install all dependencies with Homebrew: - brew install boost lua51 shapelib rapidjson + brew install boost lua shapelib rapidjson Then: make sudo make install +(System Integrity Protection on macOS prevents the manpages being installed. This isn't important: ignore the two lines saying "Operation not permitted".) + ### Ubuntu Start with: @@ -22,7 +24,7 @@ Once you've installed those, then `cd` back to your Tilemaker directory and simp make sudo make install -If it fails, check that the LIB and INC lines in the Makefile correspond with your system, then try again. +If it fails, check that the LIB and INC lines in the Makefile correspond with your system, then try again. The above lines install Lua 5.1, but you can also choose any newer version. ### Fedora diff --git a/docs/RELATIONS.md b/docs/RELATIONS.md index 6e436b68..55a0d75c 100644 --- a/docs/RELATIONS.md +++ b/docs/RELATIONS.md @@ -1,50 +1,54 @@ -## Relations +# Relations -Tilemaker has (as yet not complete) support for reading relations in the Lua process scripts. This means you can support objects like route relations when creating your vector tiles. +Tilemaker supports reading relations in the Lua process scripts. This means you can include objects like route relations when creating your vector tiles. -Note that relation support is in its early stages and behaviour may change between point versions. - -### Multipolygon relations +## Multipolygon relations Multipolygon relations are supported natively by tilemaker; you do not need to write special Lua code for them. When a multipolygon is read, tilemaker constructs the geometry as normal, and passes the tags to `way_function` just as it would a simple area. Boundary relations also have automatic handling of inner/outer ways, but are otherwise processed as relations. This means that you can choose to treat boundaries as properties on ways (see "Stage 2" below) or as complete geometries (see "Writing relation geometries"). You will typically want the properties-on-ways approach for administrative boundaries, and the complete-geometries approach for filled areas such as forests or nature reserves. -### Reading relation memberships +## Reading relation memberships You can set your Lua script so that `way_function` is able to access the relations that the way is a member of. You would use this, for example, if you wanted to read road numbers from a relation, or to colour roads that are part of a bus route differently. This is a two-stage process: first, when reading relations, indicate that these should be considered ("accepted"); then, when reading ways in `way_function`, you can access the relation tags. -#### Stage 1: accepting relations +### Stage 1: accepting relations To define which relations should be accepted, add a `relation_scan_function`: - function relation_scan_function(relation) - if relation:Find("type")=="route" and relation:Find("route")=="bicycle" then - local network = relation:Find("network") - if network=="ncn" then relation:Accept() end - end - end +```lua +function relation_scan_function() + if Find("type")=="route" and Find("route")=="bicycle" then + local network = Find("network") + if network=="ncn" then Accept() end + end +end +``` -This function takes the relation as its sole argument. Examine the tags using `relation:Find(key)` as normal. (You can also use `relation:Holds(key)` and `relation:Id()`.) If you want to use this relation, call `relation:Accept()`. +Examine the tags using `Find(key)` as normal. (You can also use `Holds(key)` and `Id()`.) If you want to use this relation, call `Accept()`. -#### Stage 2: accessing relations from ways +### Stage 2: accessing relations from ways and nodes -Now that you've accepted the relations, they will be available from `way_function`. They are accessed using an iterator (`way:NextRelation()`) which reads each relation for that way in turn, returning nil when there are no more relations available. Once you have accessed a relation with the iterator, you can read its tags with `way:FindInRelation(key)`. For example: +Now that you've accepted the relations, they will be available from `way_function` or `node_function`. They are accessed using an iterator (`NextRelation()`) which reads each relation for that way in turn, returning nil when there are no more relations available. Once you have accessed a relation with the iterator, you can read its tags with `FindInRelation(key)`. For example: - while true do - local rel = way:NextRelation() - if not rel then break end - print ("Part of route "..way:FindInRelation("ref")) - end +```lua +while true do + local rel = NextRelation() + if not rel then break end + print ("Part of route "..FindInRelation("ref")) +end +``` + +You can obtain the relation ID and role from `rel`, which is a list (a two-element Lua table). The first element (`rel[1]`) is the id, the second (`rel[2]`) the role. Remember Lua tables are 1-indexed! -(Should you need to re-read the relations, you can reset the iterator with `way:RestartRelations()`.) +Should you need to re-read the relations, you can reset the iterator with `RestartRelations()`. -### Writing relation geometries +## Writing relation geometries You can also construct complete multi-linestring geometries from relations. Use this if, for example, you want a geometry for a bike or bus route that you can show at lower zoom levels, or draw with a continuous pattern. @@ -52,19 +56,50 @@ First, make sure that you have accepted the relations using `relation_scan_funct Then write a `relation_function`, which works in the same way as `way_function` would: - function relation_function(relation) - if relation:Find("type")=="route" and relation:Find("route")=="bicycle" then - relation:Layer("bike_routes", false) - relation:Attribute("class", relation:Find("network")) - relation:Attribute("ref", relation:Find("ref")) - end - end +```lua +function relation_function() + if Find("type")=="route" and Find("route")=="bicycle" then + Layer("bike_routes", false) + Attribute("class", Find("network")) + Attribute("ref", Find("ref")) + end +end +``` + + +## Nested relations + +In some advanced circumstances you may need to deal with nested relations - for example, routes and superroutes. tilemaker provides two ways to do this. Note that you always need to `Accept()` the parent relations in `relation_scan_function` so that they're available for processing. +### Processing nested relations in relation_function -### Not supported +You can read parent relations in `relation_function` in just the same way as you would for ways and nodes (see "Stage 2" above). Just iterate through them with `NextRelation` and read their tags with `FindInRelation`. + +### Bouncing tags down + +The three main processing functions (`way_function`, `node_function`, `relation_function`) only cope with one level of parent relations for performance reasons. So if you need to work with grandparent relations, this won't work on its own. + +However, relation hierarchies aren't available in `relation_scan_function`, because not all relations have been read yet. + +Instead, you can supply an additional `relation_postscan_function`. This runs immediately after `relation_scan_function`, when all accepted relations are in memory. The concept is that you read the ancestor relations, and then use this to set tags on the child relation. These tags will then be accessible when reading the relation in `way_function` etc. + +For example, let's say you have a cycle route relation 8590454 (`type=route, route=bicycle, network=ncn, name=Loire à Vélo 3`). This has a parent relation 3199605 (`type=superroute, route=bicycle, network=ncn, name=Loire à Vélo`). You want to use the name from the latter. Write a relation_postscan_function like this: + +```lua +function relation_postscan_function() + while true do + local parent,role = NextRelation() + if not parent then break end + -- FindInRelation reads from the parent relation + -- SetTag sets on the child relation + if FindInRelation("type")=="superroute" then + local parent_name = FindInRelation("name") + SetTag("name", parent_name) + end + end +end +``` -Tilemaker does not yet support: +This will alter the relation 8590454 to have the name from the parent superroute, "Loire à Vélo". -- relation roles -- nested relations -- nodes in relations +In case of deeply nested hierarchies, tilemaker flattens them out so that `relation_postscan_function` can iterate through all parents/grandparents/etc. with `NextRelation`. The hierarchy is not preserved. diff --git a/docs/RUNNING.md b/docs/RUNNING.md index a27d83fb..c1acb6bf 100644 --- a/docs/RUNNING.md +++ b/docs/RUNNING.md @@ -33,8 +33,41 @@ to writing them out as vector tiles. This is fine for small regions, but can imp requirements for larger areas. To use on-disk storage instead, pass the `--store` argument with a path to the directory where -you want the temporary store to be created. This should be on an SSD or other fast disk. -Tilemaker will grow the store as required. +you want the temporary store to be created - on an SSD or other fast disk. This allows you +to process bigger areas even on memory-constrained systems. + +## Performance and memory tuning + +By default, tilemaker aims to balance memory usage with speed, but with a slight tilt towards +minimising memory. You can get faster runtimes, at the expense of a little more memory, by +specifying `--fast`. + +This is all most people will need to know. But if you have plentiful RAM, you can experiment +with these options. In general, options that use more memory run faster - but if you can +optimise memory such that your dataset fits entirely in RAM, this will be a big speed-up. +(`--fast` simply chooses a set of these options for you.) + +* `--compact`: Use a smaller, faster data structure for node lookups. __Note__: This requires +the .pbf to have nodes in sequential order, typically by using `osmium renumber`. +* `--no-compress-nodes` and `--no-compress-ways`: Turn off node/way compression. Increases +RAM usage but runs faster. +* `--materialize-geometries`: Generate geometries in advance when reading .pbf. Increases RAM +usage but runs faster. +* `--shard-stores`: Group temporary storage by area. Reduces RAM usage on large files (e.g. +whole planet) but runs slower. + +You can also tell tilemaker to only look at .pbf objects with certain tags. If you're making a +thematic map, this allows tilemaker to skip data it won't need. Specify this in your Lua file +like one of these three examples: + + -- Only include major roads + way_keys = {"highway=motorway", "highway=trunk", "highway=primary", "highway=secondary"}` + + -- Only include railways + way_keys = {"railway"} + + -- Include everything but not buildings + way_keys = {"~building"} ## Merging @@ -58,25 +91,10 @@ Then rerun with another .pbf, using the `--merge` flag: The second run will proceed a little more slowly due to reading in existing tiles in areas which overlap. Any OSM objects which appear in both files will be written twice. -For very large areas, you could potentially use `osmium tags-filter` to split a .pbf into several -"thematic" extracts: for example, one containing buildings, another roads, and another landuse. -Renumber each one, then run tilemaker several times with `--merge` to add one theme at a time. -This would greatly reduce memory usage. - -## Pre-split data - -Tilemaker is able to read pre-split source data, where the original .osm.pbf has already been -split into tiled areas (but not converted any further). By reducing the amount of data tilemaker -has to process at any one time, this can greatly reduce memory requirements. - -To split an .osm.pbf, use [mapsplit](https://github.com/simonpoole/mapsplit). This will output -an .msf file. We would recommend that you split the data at a low zoom level, such as 6; -tilemaker will not be able to generate vector tiles at a lower zoom level than the one you -choose for your .msf file. - -You can then run tilemaker exactly as normal, with the `--input` parameter set to your .msf -file. Source tiles will be processed one by one. Note that shapefiles will be read unsplit as -normal. +If you're very pushed for memory, you could potentially use `osmium tags-filter` to split a +.pbf into several "thematic" extracts: for example, one containing buildings, another roads, +and another landuse. Renumber each one, then run tilemaker several times with `--merge` to add +one theme at a time. ## Output messages diff --git a/docs/VECTOR_TILES.md b/docs/VECTOR_TILES.md index f169852f..2e1de838 100644 --- a/docs/VECTOR_TILES.md +++ b/docs/VECTOR_TILES.md @@ -38,7 +38,7 @@ There's no one standard schema, but that used by the OpenMapTiles project is pop Once you've generated your vector tiles, you need to render them on-screen. This is outside tilemaker's scope, but here's some pointers. -If you're serving them over the web, you'll need a server which accepts requests, and responds with the relevant tile. You'll usually do this by reading the record from the .mbtiles container (for which you can use a SQLite client library) and sending it back over HTTP. There's a Ruby example of how to do this in tilemaker's `server/` directory. Ready-made servers in other languages are available as open source, such as [mbtileserver](https://github.com/consbio/mbtileserver) (Go) and [tileserver-php](https://github.com/maptiler/tileserver-php), or you can use [tileserverless](https://github.com/geolonia/tileserverless) to serve direct from AWS. +If you're serving them over the web, you'll need a server which accepts requests, and responds with the relevant tile. You'll usually do this by reading the record from the .mbtiles container (for which you can use a SQLite client library) and sending it back over HTTP. The bundled `tilemaker-server` will do this for you, and there's also a Ruby example of how to do this in tilemaker's `server/` directory. Ready-made servers in other languages are available as open source, such as [mbtileserver](https://github.com/consbio/mbtileserver) (Go) and [tileserver-php](https://github.com/maptiler/tileserver-php), or you can use [tileserverless](https://github.com/geolonia/tileserverless) to serve direct from AWS. It's then up to the client to render the tiles. There are a few libraries that do this, but the most popular and full-featured is that developed by Mapbox, usually known as Mapbox GL. The latest versions of this are closed-source and require a Mapbox contract, but the earlier open-source version has been forked and continues as [MapLibre GL](https://github.com/maplibre), which we recommend. There's a JavaScript version as well as an iOS/Android native version. diff --git a/include/attribute_store.h b/include/attribute_store.h index 3aea19cf..dc210b96 100644 --- a/include/attribute_store.h +++ b/include/attribute_store.h @@ -10,6 +10,7 @@ #include #include #include +#include #include "pooled_string.h" #include "deque_map.h" @@ -126,7 +127,7 @@ struct AttributePair { void ensureStringIsOwned(); - static bool isHot(const std::string& keyName, const std::string& value) { + static bool isHot(const std::string& keyName, const protozero::data_view value) { // Is this pair a candidate for the hot pool? // Hot pairs are pairs that we think are likely to be re-used, like @@ -139,7 +140,8 @@ struct AttributePair { // Only strings that are IDish are eligible: only lowercase letters. bool ok = true; - for (const auto& c: value) { + for (size_t i = 0; i < value.size(); i++) { + const auto c = value.data()[i]; if (c != '-' && c != '_' && (c < 'a' || c > 'z')) return false; } @@ -183,11 +185,14 @@ struct AttributePair { #define SHARD_BITS 14 #define ATTRIBUTE_SHARDS (1 << SHARD_BITS) +class AttributeStore; class AttributePairStore { public: AttributePairStore(): finalized(false), - pairsMutex(ATTRIBUTE_SHARDS) + pairsMutex(ATTRIBUTE_SHARDS), + lookups(0), + lookupsUncached(0) { // The "hot" shard has a capacity of 64K, the others are unbounded. pairs.push_back(DequeMap(1 << 16)); @@ -202,9 +207,10 @@ class AttributePairStore { const AttributePair& getPairUnsafe(uint32_t i) const; uint32_t addPair(AttributePair& pair, bool isHot); - std::vector> pairs; private: + friend class AttributeStore; + std::vector> pairs; bool finalized; // We refer to all attribute pairs by index. // @@ -214,6 +220,8 @@ class AttributePairStore { // we suspect will be popular. It only ever has 64KB items, // so that we can reference it with a short. mutable std::vector pairsMutex; + std::atomic lookupsUncached; + std::atomic lookups; }; // AttributeSet is a set of AttributePairs @@ -398,7 +406,7 @@ struct AttributeStore { void reportSize() const; void finalize(); - void addAttribute(AttributeSet& attributeSet, std::string const &key, const std::string& v, char minzoom); + void addAttribute(AttributeSet& attributeSet, std::string const &key, const protozero::data_view v, char minzoom); void addAttribute(AttributeSet& attributeSet, std::string const &key, float v, char minzoom); void addAttribute(AttributeSet& attributeSet, std::string const &key, bool v, char minzoom); @@ -406,7 +414,8 @@ struct AttributeStore { finalized(false), sets(ATTRIBUTE_SHARDS), setsMutex(ATTRIBUTE_SHARDS), - lookups(0) { + lookups(0), + lookupsUncached(0) { } AttributeKeyStore keyStore; @@ -418,6 +427,7 @@ struct AttributeStore { mutable std::vector setsMutex; mutable std::mutex mutex; + std::atomic lookupsUncached; std::atomic lookups; }; diff --git a/include/coordinates.h b/include/coordinates.h index 3ca3b8cf..6d34ca61 100644 --- a/include/coordinates.h +++ b/include/coordinates.h @@ -19,6 +19,7 @@ typedef uint32_t ShardedNodeID; typedef uint64_t NodeID; typedef uint64_t WayID; +typedef uint64_t RelationID; typedef std::vector WayVec; diff --git a/include/deque_map.h b/include/deque_map.h index bcb4ddbc..ea57f669 100644 --- a/include/deque_map.h +++ b/include/deque_map.h @@ -95,7 +95,11 @@ class DequeMap { return -1; } - const T& at(uint32_t index) const { + inline const T& operator[](uint32_t index) const { + return objects[index]; + } + + inline const T& at(uint32_t index) const { return objects.at(index); } diff --git a/include/options_parser.h b/include/options_parser.h index 2d5caf8a..a7e15b18 100644 --- a/include/options_parser.h +++ b/include/options_parser.h @@ -29,9 +29,6 @@ namespace OptionsParser { bool uncompressedNodes = false; bool uncompressedWays = false; bool materializeGeometries = false; - // lazyGeometries is the inverse of materializeGeometries. It can be passed - // to override an implicit materializeGeometries, as in the non-store case. - bool lazyGeometries = false; bool shardStores = false; }; @@ -47,7 +44,6 @@ namespace OptionsParser { bool showHelp = false; bool verbose = false; bool mergeSqlite = false; - bool mapsplit = false; OutputMode outputMode = OutputMode::File; bool logTileTimings = false; }; diff --git a/include/osm_lua_processing.h b/include/osm_lua_processing.h index c93288b8..d193aa95 100644 --- a/include/osm_lua_processing.h +++ b/include/osm_lua_processing.h @@ -18,6 +18,9 @@ #include +class TagMap; +class SignificantTags; + // Lua extern "C" { #include "lua.h" @@ -58,12 +61,14 @@ class OsmLuaProcessing { ~OsmLuaProcessing(); // ---- Helpers provided for main routine + void handleUserSignal(int signum); // Has this object been assigned to any layers? bool empty(); // Do we have Lua routines for non-MP relations? bool canReadRelations(); + bool canPostScanRelations(); bool canWriteRelations(); // Shapefile tag remapping @@ -76,13 +81,16 @@ class OsmLuaProcessing { using tag_map_t = boost::container::flat_map; // Scan non-MP relation - bool scanRelation(WayID id, const tag_map_t &tags); + bool scanRelation(WayID id, const TagMap& tags); + // Post-scan non-MP relations + void postScanRelations(); + /// \brief We are now processing a significant node - void setNode(NodeID id, LatpLon node, const tag_map_t &tags); + bool setNode(NodeID id, LatpLon node, const TagMap& tags); /// \brief We are now processing a way - bool setWay(WayID wayId, LatpLonVec const &llVec, const tag_map_t &tags); + bool setWay(WayID wayId, LatpLonVec const &llVec, const TagMap& tags); /** \brief We are now processing a relation * (note that we store relations as ways with artificial IDs, and that @@ -93,7 +101,7 @@ class OsmLuaProcessing { const PbfReader::Relation& relation, const WayVec& outerWayVec, const WayVec& innerWayVec, - const tag_map_t& tags, + const TagMap& tags, bool isNativeMP, bool isInnerOuter ); @@ -109,6 +117,9 @@ class OsmLuaProcessing { // Get an OSM tag for a given key (or return empty string if none) const std::string Find(const std::string& key) const; + // Check if an object has any tags + bool HasTags() const; + // ---- Spatial queries called from Lua // Find intersecting shapefile layer @@ -145,7 +156,6 @@ class OsmLuaProcessing { template CorrectGeometryResult CorrectGeometry(GeometryT &geom) { -#if BOOST_VERSION >= 105800 geom::validity_failure_type failure = geom::validity_failure_type::no_failure; if (isRelation && !geom::is_valid(geom,failure)) { if (verbose) std::cout << "Relation " << originalOsmID << " has " << boost_validity_error(failure) << std::endl; @@ -165,7 +175,6 @@ class OsmLuaProcessing { } return CorrectGeometryResult::Corrected; } -#endif return CorrectGeometryResult::Valid; } @@ -174,12 +183,9 @@ class OsmLuaProcessing { void LayerAsCentroid(const std::string &layerName, kaguya::VariadicArgType nodeSources); // Set attributes in a vector tile's Attributes table - void Attribute(const std::string &key, const std::string &val); - void AttributeWithMinZoom(const std::string &key, const std::string &val, const char minzoom); - void AttributeNumeric(const std::string &key, const float val); - void AttributeNumericWithMinZoom(const std::string &key, const float val, const char minzoom); - void AttributeBoolean(const std::string &key, const bool val); - void AttributeBooleanWithMinZoom(const std::string &key, const bool val, const char minzoom); + void Attribute(const std::string &key, const protozero::data_view val, const char minzoom); + void AttributeNumeric(const std::string &key, const float val, const char minzoom); + void AttributeBoolean(const std::string &key, const bool val, const char minzoom); void MinZoom(const double z); void ZOrder(const double z); @@ -194,6 +200,7 @@ class OsmLuaProcessing { void RestartRelations(); std::string FindInRelation(const std::string &key); void Accept(); + void SetTag(const std::string &key, const std::string &value); // Write error if in verbose mode void ProcessingError(const std::string &errStr) { @@ -204,7 +211,8 @@ class OsmLuaProcessing { void setVectorLayerMetadata(const uint_least8_t layer, const std::string &key, const uint type); - std::vector GetSignificantNodeKeys(); + SignificantTags GetSignificantNodeKeys(); + SignificantTags GetSignificantWayKeys(); // ---- Cached geometries creation @@ -219,6 +227,8 @@ class OsmLuaProcessing { inline AttributeStore &getAttributeStore() { return attributeStore; } struct luaProcessingException :std::exception {}; + const TagMap* currentTags; + bool isPostScanRelation; // processing a relation in postScanRelation private: /// Internal: clear current cached state @@ -237,8 +247,13 @@ class OsmLuaProcessing { relationList.clear(); relationSubscript = -1; lastStoredGeometryId = 0; + isWay = false; + isRelation = false; + isPostScanRelation = false; } + void removeAttributeIfNeeded(const std::string& key); + const inline Point getPoint() { return Point(lon/10000000.0,latp/10000000.0); } @@ -248,6 +263,7 @@ class OsmLuaProcessing { kaguya::State luaState; bool supportsRemappingShapefiles; bool supportsReadingRelations; + bool supportsPostScanRelations; bool supportsWritingRelations; const class ShpMemTiles &shpMemTiles; class OsmMemTiles &osmMemTiles; @@ -281,8 +297,9 @@ class OsmLuaProcessing { class LayerDefinition &layers; std::vector> outputs; // All output objects that have been created - const boost::container::flat_map* currentTags; + std::vector outputKeys; const PbfReader::Relation* currentRelation; + const boost::container::flat_map* currentPostScanTags; // for postScan only const std::vector* stringTable; std::vector finalizeOutputs(); diff --git a/include/osm_store.h b/include/osm_store.h index 2b378ebb..53b21f48 100644 --- a/include/osm_store.h +++ b/include/osm_store.h @@ -19,6 +19,22 @@ extern bool verbose; class NodeStore; class WayStore; +class UsedObjects { +public: + enum class Status: bool { Disabled = false, Enabled = true }; + UsedObjects(Status status); + bool test(NodeID id); + void set(NodeID id); + void enable(); + bool enabled() const; + void clear(); + +private: + Status status; + std::vector mutex; + std::vector> ids; +}; + // A comparator for data_view so it can be used in boost's flat_map struct DataViewLessThan { bool operator()(const protozero::data_view& a, const protozero::data_view& b) const { @@ -82,52 +98,105 @@ class RelationScanStore { private: using tag_map_t = boost::container::flat_map; - std::map>> relationsForWays; - std::map>> relationsForNodes; - std::map relationTags; - mutable std::mutex mutex; + std::vector>>> relationsForWays; + std::vector>>> relationsForNodes; + std::vector> relationTags; + mutable std::vector mutex; RelationRoles relationRoles; public: - void relation_contains_way(WayID relid, WayID wayid, std::string role) { + std::map>> relationsForRelations; + + RelationScanStore(): relationsForWays(128), relationsForNodes(128), relationTags(128), mutex(128) {} + + void relation_contains_way(RelationID relid, WayID wayid, std::string role) { uint16_t roleId = relationRoles.getOrAddRole(role); - std::lock_guard lock(mutex); - relationsForWays[wayid].emplace_back(std::make_pair(relid, roleId)); + const size_t shard = wayid % mutex.size(); + std::lock_guard lock(mutex[shard]); + relationsForWays[shard][wayid].emplace_back(std::make_pair(relid, roleId)); } - void relation_contains_node(WayID relid, NodeID nodeId, std::string role) { + void relation_contains_node(RelationID relid, NodeID nodeId, std::string role) { uint16_t roleId = relationRoles.getOrAddRole(role); - std::lock_guard lock(mutex); - relationsForNodes[nodeId].emplace_back(std::make_pair(relid, roleId)); + const size_t shard = nodeId % mutex.size(); + std::lock_guard lock(mutex[shard]); + relationsForNodes[shard][nodeId].emplace_back(std::make_pair(relid, roleId)); } - void store_relation_tags(WayID relid, const tag_map_t &tags) { - std::lock_guard lock(mutex); - relationTags[relid] = tags; + void relation_contains_relation(RelationID relid, RelationID relationId, std::string role) { + uint16_t roleId = relationRoles.getOrAddRole(role); + std::lock_guard lock(mutex[0]); + relationsForRelations[relationId].emplace_back(std::make_pair(relid, roleId)); + } + void store_relation_tags(RelationID relid, const tag_map_t &tags) { + const size_t shard = relid % mutex.size(); + std::lock_guard lock(mutex[shard]); + relationTags[shard][relid] = tags; + } + void set_relation_tag(RelationID relid, const std::string &key, const std::string &value) { + const size_t shard = relid % mutex.size(); + std::lock_guard lock(mutex[shard]); + relationTags[shard][relid][key] = value; } bool way_in_any_relations(WayID wayid) { - return relationsForWays.find(wayid) != relationsForWays.end(); + const size_t shard = wayid % mutex.size(); + return relationsForWays[shard].find(wayid) != relationsForWays[shard].end(); } bool node_in_any_relations(NodeID nodeId) { - return relationsForNodes.find(nodeId) != relationsForNodes.end(); + const size_t shard = nodeId % mutex.size(); + return relationsForNodes[shard].find(nodeId) != relationsForNodes[shard].end(); + } + bool relation_in_any_relations(RelationID relId) { + return relationsForRelations.find(relId) != relationsForRelations.end(); } std::string getRole(uint16_t roleId) const { return relationRoles.getRole(roleId); } const std::vector>& relations_for_way(WayID wayid) { - return relationsForWays[wayid]; + const size_t shard = wayid % mutex.size(); + return relationsForWays[shard][wayid]; + } + const std::vector>& relations_for_node(NodeID nodeId) { + const size_t shard = nodeId % mutex.size(); + return relationsForNodes[shard][nodeId]; + } + const std::vector>& relations_for_relation(RelationID relId) { + return relationsForRelations[relId]; + } + bool has_relation_tags(RelationID relId) { + const size_t shard = relId % mutex.size(); + return relationTags[shard].find(relId) != relationTags[shard].end(); + } + + const tag_map_t& relation_tags(RelationID relId) { + const size_t shard = relId % mutex.size(); + return relationTags[shard][relId]; } - const std::vector>& relations_for_node(NodeID nodeId) { - return relationsForNodes[nodeId]; + // return all the parent relations (and their parents &c.) for a given relation + std::vector> relations_for_relation_with_parents(RelationID relId) { + std::vector relationsToDo; + std::set relationsDone; + std::vector> out; + relationsToDo.emplace_back(relId); + // check parents in turn, pushing onto the stack if necessary + while (!relationsToDo.empty()) { + RelationID rel = relationsToDo.back(); + relationsToDo.pop_back(); + // check it's not already been added + if (relationsDone.find(rel) != relationsDone.end()) continue; + relationsDone.insert(rel); + // add all its parents + for (auto rp : relationsForRelations[rel]) { + out.emplace_back(rp); + relationsToDo.emplace_back(rp.first); + } + } + return out; } - std::string get_relation_tag(WayID relid, const std::string &key) { - auto it = relationTags.find(relid); - if (it==relationTags.end()) return ""; + std::string get_relation_tag(RelationID relid, const std::string &key) { + const size_t shard = relid % mutex.size(); + auto it = relationTags[shard].find(relid); + if (it==relationTags[shard].end()) return ""; auto jt = it->second.find(key); if (jt==it->second.end()) return ""; return jt->second; } - void clear() { - std::lock_guard lock(mutex); - relationsForWays.clear(); - relationTags.clear(); - } }; @@ -202,8 +271,18 @@ class OSMStore UsedWays used_ways; public: - - OSMStore(NodeStore& nodes, WayStore& ways): nodes(nodes), ways(ways) + UsedObjects usedNodes; + UsedObjects usedRelations; + + OSMStore(NodeStore& nodes, WayStore& ways): + nodes(nodes), + ways(ways), + // We only track usedNodes if way_keys is present; a node is used if it's + // a member of a way used by a used relation, or a way that meets the way_keys + // criteria. + usedNodes(UsedObjects::Status::Disabled), + // A relation is used only if it was previously accepted from relation_scan_function + usedRelations(UsedObjects::Status::Enabled) { reopen(); } diff --git a/include/pbf_processor.h b/include/pbf_processor.h index d79d1ca5..daaf39d0 100644 --- a/include/pbf_processor.h +++ b/include/pbf_processor.h @@ -8,7 +8,9 @@ #include #include #include "osm_store.h" +#include "significant_tags.h" #include "pbf_reader.h" +#include "tag_map.h" #include class OsmLuaProcessing; @@ -43,7 +45,7 @@ struct IndexedBlockMetadata: BlockMetadata { class PbfProcessor { public: - enum class ReadPhase { Nodes = 1, Ways = 2, Relations = 4, RelationScan = 8 }; + enum class ReadPhase { Nodes = 1, Ways = 2, Relations = 4, RelationScan = 8, WayScan = 16 }; PbfProcessor(OSMStore &osmStore); @@ -53,7 +55,8 @@ class PbfProcessor int ReadPbfFile( uint shards, bool hasSortTypeThenID, - const std::unordered_set& nodeKeys, + const SignificantTags& nodeKeys, + const SignificantTags& wayKeys, unsigned int threadNum, const pbfreader_generate_stream& generate_stream, const pbfreader_generate_output& generate_output, @@ -62,16 +65,12 @@ class PbfProcessor ); // Read tags into a map from a way/node/relation - using tag_map_t = boost::container::flat_map; template - void readTags(T& pbfObject, const PbfReader::PrimitiveBlock& pb, tag_map_t& tags) { - tags.reserve(pbfObject.keys.size()); + void readTags(T &pbfObject, PbfReader::PrimitiveBlock const &pb, TagMap& tags) { for (uint n=0; n < pbfObject.keys.size(); n++) { auto keyIndex = pbfObject.keys[n]; auto valueIndex = pbfObject.vals[n]; - protozero::data_view key = pb.stringTable[keyIndex]; - protozero::data_view value = pb.stringTable[valueIndex]; - tags[key] = value; + tags.addTag(pb.stringTable[keyIndex], pb.stringTable[valueIndex]); } } @@ -80,28 +79,32 @@ class PbfProcessor std::istream &infile, OsmLuaProcessing &output, const BlockMetadata& blockMetadata, - const std::unordered_set& nodeKeys, + const SignificantTags& nodeKeys, + const SignificantTags& wayKeys, bool locationsOnWays, ReadPhase phase, uint shard, uint effectiveShard ); - bool ReadNodes(OsmLuaProcessing& output, PbfReader::PrimitiveGroup& pg, const PbfReader::PrimitiveBlock& pb, const std::unordered_set& nodeKeyPositions); + bool ReadNodes(OsmLuaProcessing& output, PbfReader::PrimitiveGroup& pg, const PbfReader::PrimitiveBlock& pb, const SignificantTags& nodeKeys); bool ReadWays( OsmLuaProcessing& output, PbfReader::PrimitiveGroup& pg, const PbfReader::PrimitiveBlock& pb, + const SignificantTags& wayKeys, bool locationsOnWays, uint shard, uint effectiveShards ); - bool ScanRelations(OsmLuaProcessing& output, PbfReader::PrimitiveGroup& pg, const PbfReader::PrimitiveBlock& pb); + bool ScanWays(OsmLuaProcessing& output, PbfReader::PrimitiveGroup& pg, const PbfReader::PrimitiveBlock& pb, const SignificantTags& wayKeys); + bool ScanRelations(OsmLuaProcessing& output, PbfReader::PrimitiveGroup& pg, const PbfReader::PrimitiveBlock& pb, const SignificantTags& wayKeys); bool ReadRelations( OsmLuaProcessing& output, PbfReader::PrimitiveGroup& pg, const PbfReader::PrimitiveBlock& pb, const BlockMetadata& blockMetadata, + const SignificantTags& wayKeys, uint shard, uint effectiveShards ); diff --git a/include/pooled_string.h b/include/pooled_string.h index 9e280855..a130f737 100644 --- a/include/pooled_string.h +++ b/include/pooled_string.h @@ -32,6 +32,7 @@ #include #include #include +#include namespace PooledStringNS { class PooledString { @@ -40,9 +41,10 @@ namespace PooledStringNS { PooledString(const std::string& str); - // Create a std string - only valid so long as the string that is - // pointed to is valid. - PooledString(const std::string* str); + // Wrap a protozero::data_view - only valid so long as the + // data_view that is pointed to is valid; call ensureStringIsOwned + // if you need to persist the string. + PooledString(const protozero::data_view* str); size_t size() const; bool operator<(const PooledString& other) const; bool operator==(const PooledString& other) const; diff --git a/include/significant_tags.h b/include/significant_tags.h new file mode 100644 index 00000000..770f31d3 --- /dev/null +++ b/include/significant_tags.h @@ -0,0 +1,39 @@ +#ifndef SIGNIFICANT_TAGS_H +#define SIGNIFICANT_TAGS_H + +#include +#include + +class TagMap; +// Data structures to permit users to express filters on which nodes/ways +// to be accepted. +// +// Filters are of the shape: [~]key-name[=value-name] +// +// When a tilde is present, the filter's meaning is inverted. + +struct TagFilter { + bool accept; + std::string key; + std::string value; + + bool operator==(const TagFilter& other) const { + return accept == other.accept && key == other.key && value == other.value; + } +}; + +class SignificantTags { +public: + SignificantTags(); + SignificantTags(std::vector rawTags); + bool filter(const TagMap& tags) const; + + static TagFilter parseFilter(std::string rawTag); + bool enabled() const; + +private: + bool enabled_; + std::vector filters; +}; + +#endif diff --git a/include/tag_map.h b/include/tag_map.h new file mode 100644 index 00000000..be5fa97c --- /dev/null +++ b/include/tag_map.h @@ -0,0 +1,75 @@ +#ifndef _TAG_MAP_H +#define _TAG_MAP_H + +#include +#include +#include +#include + +// We track tags in a special structure, which enables some tricks when +// doing Lua interop. +// +// The alternative is a std::map - but often, our map is quite small. +// It's preferable to have a small set of vectors and do linear search. +// +// Further, we can avoid passing std::string from Lua -> C++ in some cases +// by first checking to see if the string we would have passed is already +// stored in our tag map, and passing a reference to its location. + +// Assumptions: +// 1. Not thread-safe +// This is OK because we have 1 instance of OsmLuaProcessing per thread. +// 2. Lifetime of map is less than lifetime of keys/values that are passed +// This is true since the strings are owned by the protobuf block reader +// 3. Max number of tag values will fit in a short +// OSM limit is 5,000 tags per object +struct Tag { + protozero::data_view key; + protozero::data_view value; +}; + +class TagMap { +public: + TagMap(); + void reset(); + + bool empty() const; + void addTag(const protozero::data_view& key, const protozero::data_view& value); + + // Return -1 if key not found, else return its keyLoc. + int64_t getKey(const char* key, size_t size) const; + + // Return -1 if value not found, else return its keyLoc. + int64_t getValue(const char* key, size_t size) const; + + const protozero::data_view* getValueFromKey(uint32_t keyLoc) const; + const protozero::data_view* getValue(uint32_t valueLoc) const; + + boost::container::flat_map exportToBoostMap() const; + + struct Iterator { + const TagMap& map; + size_t shard = 0; + size_t offset = 0; + + bool operator!=(const Iterator& other) const; + void operator++(); + Tag operator*() const; + }; + + Iterator begin() const; + Iterator end() const; + +private: + uint32_t ensureString( + std::vector>& vector, + const protozero::data_view& value + ); + + + std::vector> keys; + std::vector> key2value; + std::vector> values; +}; + +#endif _TAG_MAP_H diff --git a/include/tile_data.h b/include/tile_data.h index 6b59ee3f..b78463e2 100644 --- a/include/tile_data.h +++ b/include/tile_data.h @@ -364,6 +364,8 @@ class TileDataSource { ClipCache multiPolygonClipCache; ClipCache multiLinestringClipCache; + std::deque>> pendingSmallIndexObjects; + public: TileDataSource(size_t threadNum, unsigned int baseZoom, bool includeID); @@ -391,6 +393,8 @@ class TileDataSource { ); void addObjectToSmallIndex(const TileCoordinates& index, const OutputObject& oo, uint64_t id); + void addObjectToSmallIndex(const TileCoordinates& index, const OutputObject& oo, uint64_t id, bool needsLock); + void addObjectToSmallIndexUnsafe(const TileCoordinates& index, const OutputObject& oo, uint64_t id); void addObjectToLargeIndex(const Box& envelope, const OutputObject& oo, uint64_t id) { std::lock_guard lock(mutex); diff --git a/resources/config-example.json b/resources/config-example.json index 2c8c1b54..36a4da0e 100644 --- a/resources/config-example.json +++ b/resources/config-example.json @@ -1,25 +1,21 @@ { "layers": { - "waterway": { "minzoom": 11, "maxzoom": 14 }, - "transportation": { "minzoom": 12, "maxzoom": 14, "simplify_below": 13, "simplify_level": 0.0001, "simplify_ratio": 2.0 }, - "building": { "minzoom": 14, "maxzoom": 14 }, - "poi": { "minzoom": 13, "maxzoom": 14 } + "place": { "minzoom": 0, "maxzoom": 14 }, + "poi": { "minzoom": 12, "maxzoom": 14 }, + "transportation": { "minzoom": 4, "maxzoom": 14, "simplify_below": 13, "simplify_level": 0.0003, "simplify_ratio": 2.0 }, + "transportation_name": { "minzoom": 10, "maxzoom": 14 }, + "building": { "minzoom": 13, "maxzoom": 14 }, + "water": { "minzoom": 6, "maxzoom": 14, "simplify_below": 12, "simplify_level": 0.0003, "simplify_ratio": 2.0 }, + "waterway": { "minzoom": 6, "maxzoom": 14, "simplify_below": 12, "simplify_level": 0.0003, "simplify_ratio": 2.0 } }, "settings": { - "minzoom": 12, + "minzoom": 4, "maxzoom": 14, "basezoom": 14, "include_ids": false, "name": "Tilemaker example", "version": "0.1", "description": "Sample vector tiles for Tilemaker", - "compress": "gzip", - "metadata": { - "json": { "vector_layers": [ - { "id": "transportation", "description": "transportation", "fields": {"class":"String"}}, - { "id": "waterway", "description": "waterway", "fields": {"class":"String"}}, - { "id": "building", "description": "building", "fields": {}} - ] } - } + "compress": "gzip" } } diff --git a/resources/process-coastline.lua b/resources/process-coastline.lua index 5e2aca8e..b49eeee5 100644 --- a/resources/process-coastline.lua +++ b/resources/process-coastline.lua @@ -10,10 +10,10 @@ function exit_function() end node_keys = {} -function node_function(node) +function node_function() end -function way_function(way) +function way_function() end -- Remap coastlines diff --git a/resources/process-debug.lua b/resources/process-debug.lua index ea594c19..e1c8e62f 100644 --- a/resources/process-debug.lua +++ b/resources/process-debug.lua @@ -45,36 +45,36 @@ aerodromeValues = Set { "international", "public", "regional", "military", "priv -- Process node tags node_keys = { "amenity", "shop", "sport", "tourism", "place", "office", "natural", "addr:housenumber", "aeroway" } -function node_function(node) +function node_function() -- Write 'aerodrome_label' - local aeroway = node:Find("aeroway") + local aeroway = Find("aeroway") if aeroway == "aerodrome" then - node:Layer("aerodrome_label", false) - SetNameAttributes(node) - node:Attribute("iata", node:Find("iata")) - SetEleAttributes(node) - node:Attribute("icao", node:Find("icao")) + Layer("aerodrome_label", false) + SetNameAttributes() + Attribute("iata", Find("iata")) + SetEleAttributes() + Attribute("icao", Find("icao")) - local aerodrome_value = node:Find("aerodrome") + local aerodrome_value = Find("aerodrome") local class if aerodromeValues[aerodrome_value] then class = aerodrome_value else class = "other" end - node:Attribute("class", class) + Attribute("class", class) end -- Write 'housenumber' - local housenumber = node:Find("addr:housenumber") + local housenumber = Find("addr:housenumber") if housenumber~="" then - node:Layer("housenumber", false) - node:Attribute("housenumber", housenumber) + Layer("housenumber", false) + Attribute("housenumber", housenumber) end -- Write 'place' -- note that OpenMapTiles has a rank for countries (1-3), states (1-6) and cities (1-10+); -- we could potentially approximate it for cities based on the population tag - local place = node:Find("place") + local place = Find("place") if place ~= "" then local rank = nil local mz = 13 - local pop = tonumber(node:Find("population")) or 0 + local pop = tonumber(Find("population")) or 0 if place == "continent" then mz=2 elseif place == "country" then mz=3; rank=1 @@ -90,31 +90,31 @@ function node_function(node) elseif place == "locality" then mz=13 end - node:Layer("place", false) - node:Attribute("class", place) - node:MinZoom(mz) - if rank then node:AttributeNumeric("rank", rank) end - SetNameAttributes(node) + Layer("place", false) + Attribute("class", place) + MinZoom(mz) + if rank then AttributeNumeric("rank", rank) end + SetNameAttributes() return end -- Write 'poi' - local rank, class, subclass = GetPOIRank(node) + local rank, class, subclass = GetPOIRank() if rank then WritePOI(node,class,subclass,rank) end -- Write 'mountain_peak' and 'water_name' - local natural = node:Find("natural") + local natural = Find("natural") if natural == "peak" or natural == "volcano" then - node:Layer("mountain_peak", false) - SetEleAttributes(node) - node:AttributeNumeric("rank", 1) - node:Attribute("class", natural) - SetNameAttributes(node) + Layer("mountain_peak", false) + SetEleAttributes() + AttributeNumeric("rank", 1) + Attribute("class", natural) + SetNameAttributes() return end if natural == "bay" then - node:Layer("water_name", false) - SetNameAttributes(node) + Layer("water_name", false) + SetNameAttributes() return end end @@ -196,33 +196,33 @@ waterClasses = Set { "river", "riverbank", "stream", "canal", "drain", "ditch waterwayClasses = Set { "stream", "river", "canal", "drain", "ditch" } -function way_function(way) - local highway = way:Find("highway") - local waterway = way:Find("waterway") - local water = way:Find("water") - local building = way:Find("building") - local natural = way:Find("natural") - local historic = way:Find("historic") - local landuse = way:Find("landuse") - local leisure = way:Find("leisure") - local amenity = way:Find("amenity") - local aeroway = way:Find("aeroway") - local railway = way:Find("railway") - local sport = way:Find("sport") - local shop = way:Find("shop") - local tourism = way:Find("tourism") - local man_made = way:Find("man_made") - local isClosed = way:IsClosed() - local housenumber = way:Find("addr:housenumber") +function way_function() + local highway = Find("highway") + local waterway = Find("waterway") + local water = Find("water") + local building = Find("building") + local natural = Find("natural") + local historic = Find("historic") + local landuse = Find("landuse") + local leisure = Find("leisure") + local amenity = Find("amenity") + local aeroway = Find("aeroway") + local railway = Find("railway") + local sport = Find("sport") + local shop = Find("shop") + local tourism = Find("tourism") + local man_made = Find("man_made") + local isClosed = IsClosed() + local housenumber = Find("addr:housenumber") local write_name = false - local construction = way:Find("construction") + local construction = Find("construction") -- Miscellaneous preprocessing - if way:Find("disused") == "yes" then return end + if Find("disused") == "yes" then return end if highway == "proposed" then return end if aerowayBuildings[aeroway] then building="yes"; aeroway="" end if landuse == "field" then landuse = "farmland" end - if landuse == "meadow" and way:Find("meadow")=="agricultural" then landuse="farmland" end + if landuse == "meadow" and Find("meadow")=="agricultural" then landuse="farmland" end -- Roads ('transportation' and 'transportation_name', plus 'transportation_name_detail') if highway~="" then @@ -235,33 +235,33 @@ function way_function(way) if trackValues[highway] then h = "track"; layer="transportation_detail" end if pathValues[highway] then h = "path" ; layer="transportation_detail" end if h=="service" then layer="transportation_detail" end - way:Layer(layer, false) - way:Attribute("class", h) - SetBrunnelAttributes(way) + Layer(layer, false) + Attribute("class", h) + SetBrunnelAttributes() -- Construction if highway == "construction" then if constructionValues[construction] then - way:Attribute("class", construction .. "_construction") + Attribute("class", construction .. "_construction") else - way:Attribute("class", "minor_construction") + Attribute("class", "minor_construction") end end -- Service - local service = way:Find("service") - if highway == "service" and service ~="" then way:Attribute("service", service) end + local service = Find("service") + if highway == "service" and service ~="" then Attribute("service", service) end -- Links (ramp) if linkValues[highway] then splitHighway = split(highway, "_") highway = splitHighway[1] - way:AttributeNumeric("ramp",1) + AttributeNumeric("ramp",1) end - local oneway = way:Find("oneway") + local oneway = Find("oneway") if oneway == "yes" or oneway == "1" then - way:AttributeNumeric("oneway",1) + AttributeNumeric("oneway",1) end if oneway == "-1" then -- **** TODO @@ -269,115 +269,115 @@ function way_function(way) -- Write names if layer == "motorway" or layer == "trunk" then - way:Layer("transportation_name", false) + Layer("transportation_name", false) elseif h == "minor" or h == "track" or h == "path" or h == "service" then - way:Layer("transportation_name_detail", false) + Layer("transportation_name_detail", false) else - way:Layer("transportation_name_mid", false) + Layer("transportation_name_mid", false) end - SetNameAttributes(way) - way:Attribute("class",h) - way:Attribute("network","road") -- **** needs fixing - if h~=highway then way:Attribute("subclass",highway) end - local ref = way:Find("ref") + SetNameAttributes() + Attribute("class",h) + Attribute("network","road") -- **** needs fixing + if h~=highway then Attribute("subclass",highway) end + local ref = Find("ref") if ref~="" then - way:Attribute("ref",ref) - way:AttributeNumeric("ref_length",ref:len()) + Attribute("ref",ref) + AttributeNumeric("ref_length",ref:len()) end end -- Railways ('transportation' and 'transportation_name', plus 'transportation_name_detail') if railway~="" then - way:Layer("transportation", false) - way:Attribute("class", railway) + Layer("transportation", false) + Attribute("class", railway) - way:Layer("transportation_name", false) - SetNameAttributes(way) - way:MinZoom(14) - way:Attribute("class", "rail") + Layer("transportation_name", false) + SetNameAttributes() + MinZoom(14) + Attribute("class", "rail") end -- 'Aeroway' if aeroway~="" then - way:Layer("aeroway", isClosed) - way:Attribute("class",aeroway) - way:Attribute("ref",way:Find("ref")) + Layer("aeroway", isClosed) + Attribute("class",aeroway) + Attribute("ref",Find("ref")) write_name = true end -- 'aerodrome_label' if aeroway=="aerodrome" then - way:LayerAsCentroid("aerodrome_label") - SetNameAttributes(way) - way:Attribute("iata", way:Find("iata")) - SetEleAttributes(way) - way:Attribute("icao", way:Find("icao")) + LayerAsCentroid("aerodrome_label") + SetNameAttributes() + Attribute("iata", Find("iata")) + SetEleAttributes() + Attribute("icao", Find("icao")) - local aerodrome = way:Find(aeroway) + local aerodrome = Find(aeroway) local class if aerodromeValues[aerodrome] then class = aerodrome else class = "other" end - way:Attribute("class", class) + Attribute("class", class) end -- Set 'waterway' and associated if waterwayClasses[waterway] and not isClosed then - if waterway == "river" and way:Holds("name") then - way:Layer("waterway", false) + if waterway == "river" and Holds("name") then + Layer("waterway", false) else - way:Layer("waterway_detail", false) + Layer("waterway_detail", false) end - if way:Find("intermittent")=="yes" then way:AttributeNumeric("intermittent", 1) else way:AttributeNumeric("intermittent", 0) end - way:Attribute("class", waterway) - SetNameAttributes(way) - SetBrunnelAttributes(way) - elseif waterway == "boatyard" then way:Layer("landuse", isClosed); way:Attribute("class", "industrial") - elseif waterway == "dam" then way:Layer("building",isClosed) - elseif waterway == "fuel" then way:Layer("landuse", isClosed); way:Attribute("class", "industrial") + if Find("intermittent")=="yes" then AttributeNumeric("intermittent", 1) else AttributeNumeric("intermittent", 0) end + Attribute("class", waterway) + SetNameAttributes() + SetBrunnelAttributes() + elseif waterway == "boatyard" then Layer("landuse", isClosed); Attribute("class", "industrial") + elseif waterway == "dam" then Layer("building",isClosed) + elseif waterway == "fuel" then Layer("landuse", isClosed); Attribute("class", "industrial") end -- Set names on rivers if waterwayClasses[waterway] and not isClosed then - if waterway == "river" and way:Holds("name") then - way:Layer("water_name", false) + if waterway == "river" and Holds("name") then + Layer("water_name", false) else - way:Layer("water_name_detail", false) - way:MinZoom(14) + Layer("water_name_detail", false) + MinZoom(14) end - way:Attribute("class", waterway) - SetNameAttributes(way) + Attribute("class", waterway) + SetNameAttributes() end -- Set 'building' and associated if building~="" then - way:Layer("building", true) - SetMinZoomByArea(way) + Layer("building", true) + SetMinZoomByArea() end -- Set 'housenumber' if housenumber~="" then - way:LayerAsCentroid("housenumber", false) - way:Attribute("housenumber", housenumber) + LayerAsCentroid("housenumber", false) + Attribute("housenumber", housenumber) end -- Set 'water' if natural=="water" or natural=="bay" or leisure=="swimming_pool" or landuse=="reservoir" or landuse=="basin" or waterClasses[waterway] then - if way:Find("covered")=="yes" or not isClosed then return end + if Find("covered")=="yes" or not isClosed then return end local class="lake"; if natural=="bay" then class="ocean" elseif waterway~="" then class="river" end - way:Layer("water",true) --- SetMinZoomByArea(way) - way:Attribute("class",class) + Layer("water",true) +-- SetMinZoomByArea() + Attribute("class",class) - if way:Find("intermittent")=="yes" then way:Attribute("intermittent",1) end + if Find("intermittent")=="yes" then Attribute("intermittent",1) end -- we only want to show the names of actual lakes not every man-made basin that probably doesn't even have a name other than "basin" -- examples for which we don't want to show a name: -- https://www.openstreetmap.org/way/25958687 -- https://www.openstreetmap.org/way/27201902 -- https://www.openstreetmap.org/way/25309134 -- https://www.openstreetmap.org/way/24579306 - if way:Holds("name") and natural=="water" and water ~= "basin" and water ~= "wastewater" then - way:LayerAsCentroid("water_name_detail") - SetNameAttributes(way) --- SetMinZoomByArea(way) - way:Attribute("class", class) + if Holds("name") and natural=="water" and water ~= "basin" and water ~= "wastewater" then + LayerAsCentroid("water_name_detail") + SetNameAttributes() +-- SetMinZoomByArea() + Attribute("class", class) end return -- in case we get any landuse processing @@ -388,11 +388,11 @@ function way_function(way) if l=="" then l=natural end if l=="" then l=leisure end if landcoverKeys[l] then - way:Layer("landcover", true) - SetMinZoomByArea(way) - way:Attribute("class", landcoverKeys[l]) - if l=="wetland" then way:Attribute("subclass", way:Find("wetland")) - else way:Attribute("subclass", l) end + Layer("landcover", true) + SetMinZoomByArea() + Attribute("class", landcoverKeys[l]) + if l=="wetland" then Attribute("subclass", Find("wetland")) + else Attribute("subclass", l) end write_name = true -- Set 'landuse' @@ -400,26 +400,26 @@ function way_function(way) if l=="" then l=amenity end if l=="" then l=tourism end if landuseKeys[l] then - way:Layer("landuse", true) - way:Attribute("class", l) + Layer("landuse", true) + Attribute("class", l) write_name = true end end -- Parks - if boundary=="national_park" then way:Layer("park",true); way:Attribute("class",boundary); SetNameAttributes(way) - elseif leisure=="nature_reserve" then way:Layer("park",true); way:Attribute("class",leisure ); SetNameAttributes(way) end + if boundary=="national_park" then Layer("park",true); Attribute("class",boundary); SetNameAttributes() + elseif leisure=="nature_reserve" then Layer("park",true); Attribute("class",leisure ); SetNameAttributes() end -- POIs ('poi' and 'poi_detail') - local rank, class, subclass = GetPOIRank(way) + local rank, class, subclass = GetPOIRank() if rank then WritePOI(way,class,subclass,rank); return end -- Catch-all - if (building~="" or write_name) and way:Holds("name") then - way:LayerAsCentroid("poi_detail") - SetNameAttributes(way) + if (building~="" or write_name) and Holds("name") then + LayerAsCentroid("poi_detail") + SetNameAttributes() if write_name then rank=6 else rank=25 end - way:AttributeNumeric("rank", rank) + AttributeNumeric("rank", rank) end end @@ -435,65 +435,67 @@ end function WritePOI(obj,class,subclass,rank) local layer = "poi" if rank>4 then layer="poi_detail" end - obj:LayerAsCentroid(layer) + LayerAsCentroid(layer) SetNameAttributes(obj) - obj:AttributeNumeric("rank", rank) - obj:Attribute("class", class) - obj:Attribute("subclass", subclass) + AttributeNumeric("rank", rank) + Attribute("class", class) + Attribute("subclass", subclass) end -- Set name attributes on any object function SetNameAttributes(obj) - local name = obj:Find("name"), main_written = name, iname + local name = Find("name") + local main_written = name + local iname -- if we have a preferred language, then write that (if available), and additionally write the base name tag - if preferred_language and obj:Holds("name:"..preferred_language) then - iname = obj:Find("name:"..preferred_language) + if preferred_language and Holds("name:"..preferred_language) then + iname = Find("name:"..preferred_language) print("Found "..preferred_language..": "..iname) - obj:Attribute(preferred_language_attribute, iname) + Attribute(preferred_language_attribute, iname) if iname~=name and default_language_attribute then - obj:Attribute(default_language_attribute, name) + Attribute(default_language_attribute, name) else main_written = iname end else - obj:Attribute(preferred_language_attribute, name) + Attribute(preferred_language_attribute, name) end -- then set any additional languages for i,lang in ipairs(additional_languages) do - iname = obj:Find("name:"..lang) + iname = Find("name:"..lang) if iname=="" then iname=name end - if iname~=main_written then obj:Attribute("name:"..lang, iname) end + if iname~=main_written then Attribute("name:"..lang, iname) end end end -- Set ele and ele_ft on any object function SetEleAttributes(obj) - local ele = obj:Find("ele") + local ele = Find("ele") if ele ~= "" then local meter = math.floor(tonumber(ele) or 0) local feet = math.floor(meter * 3.2808399) - obj:AttributeNumeric("ele", meter) - obj:AttributeNumeric("ele_ft", feet) + AttributeNumeric("ele", meter) + AttributeNumeric("ele_ft", feet) end end function SetBrunnelAttributes(obj) - if obj:Find("bridge") == "yes" then obj:Attribute("brunnel", "bridge") - elseif obj:Find("tunnel") == "yes" then obj:Attribute("brunnel", "tunnel") - elseif obj:Find("ford") == "yes" then obj:Attribute("brunnel", "ford") + if Find("bridge") == "yes" then Attribute("brunnel", "bridge") + elseif Find("tunnel") == "yes" then Attribute("brunnel", "tunnel") + elseif Find("ford") == "yes" then Attribute("brunnel", "ford") end end -- Set minimum zoom level by area -function SetMinZoomByArea(way) - local area=way:Area() - if area>ZRES5^2 then way:MinZoom(6) - elseif area>ZRES6^2 then way:MinZoom(7) - elseif area>ZRES7^2 then way:MinZoom(8) - elseif area>ZRES8^2 then way:MinZoom(9) - elseif area>ZRES9^2 then way:MinZoom(10) - elseif area>ZRES10^2 then way:MinZoom(11) - elseif area>ZRES11^2 then way:MinZoom(12) - elseif area>ZRES12^2 then way:MinZoom(13) - else way:MinZoom(14) end +function SetMinZoomByArea() + local area=Area() + if area>ZRES5^2 then MinZoom(6) + elseif area>ZRES6^2 then MinZoom(7) + elseif area>ZRES7^2 then MinZoom(8) + elseif area>ZRES8^2 then MinZoom(9) + elseif area>ZRES9^2 then MinZoom(10) + elseif area>ZRES10^2 then MinZoom(11) + elseif area>ZRES11^2 then MinZoom(12) + elseif area>ZRES12^2 then MinZoom(13) + else MinZoom(14) end end -- Calculate POIs (typically rank 1-4 go to 'poi' z12-14, rank 5+ to 'poi_detail' z14) @@ -503,8 +505,8 @@ function GetPOIRank(obj) -- Can we find the tag? for k,list in pairs(poiTags) do - if list[obj:Find(k)] then - v = obj:Find(k) -- k/v are the OSM tag pair + if list[Find(k)] then + v = Find(k) -- k/v are the OSM tag pair class = poiClasses[v] or v rank = poiClassRanks[class] or 25 return rank, class, v @@ -512,7 +514,7 @@ function GetPOIRank(obj) end -- Catch-all for shops - local shop = obj:Find("shop") + local shop = Find("shop") if shop~="" then return poiClassRanks['shop'], "shop", shop end -- Nothing found diff --git a/resources/process-example.lua b/resources/process-example.lua index 41b461df..cf3266c2 100644 --- a/resources/process-example.lua +++ b/resources/process-example.lua @@ -1,46 +1,105 @@ --- Nodes will only be processed if one of these keys is present +--[[ -node_keys = { "amenity", "shop" } + A simple example tilemaker configuration, intended to illustrate how it + works and to act as a starting point for your own configurations. --- Initialize Lua logic + The basic principle is: + - read OSM tags with Find(key) + - write to vector tile layers with Layer(layer_name) + - add attributes with Attribute(field, value) -function init_function() -end + (This is a very basic subset of the OpenMapTiles schema. Don't take much + notice of the "class" attribute, that's an OMT implementation thing which + is just here to get them to show up with the default style.) + + It doesn't do much filtering by zoom level - all the roads appear all the + time. If you want a practice project, try fixing that! + + You can view your output with tilemaker-server: + + tilemaker-server /path/to/your.mbtiles --static server/static + +]]-- + + +-- Nodes will only be processed if one of these keys is present + +node_keys = { "amenity", "historic", "leisure", "place", "shop", "tourism" } --- Finalize Lua logic() -function exit_function() -end -- Assign nodes to a layer, and set attributes, based on OSM tags function node_function(node) - local amenity = node:Find("amenity") - local shop = node:Find("shop") + -- POIs go to a "poi" layer (we just look for amenity and shop here) + local amenity = Find("amenity") + local shop = Find("shop") if amenity~="" or shop~="" then - node:Layer("poi", false) - if amenity~="" then node:Attribute("class",amenity) - else node:Attribute("class",shop) end - node:Attribute("name", node:Find("name")) + Layer("poi") + if amenity~="" then Attribute("class",amenity) + else Attribute("class",shop) end + Attribute("name:latin", Find("name")) + AttributeNumeric("rank", 3) + end + + -- Places go to a "place" layer + local place = Find("place") + if place~="" then + Layer("place") + Attribute("class", place) + Attribute("name:latin", Find("name")) + if place=="city" then + AttributeNumeric("rank", 4) + MinZoom(3) + elseif place=="town" then + AttributeNumeric("rank", 6) + MinZoom(6) + else + AttributeNumeric("rank", 9) + MinZoom(10) + end end end --- Similarly for ways -function way_function(way) - local highway = way:Find("highway") - local waterway = way:Find("waterway") - local building = way:Find("building") +-- Assign ways to a layer, and set attributes, based on OSM tags + +function way_function() + local highway = Find("highway") + local waterway = Find("waterway") + local building = Find("building") + + -- Roads if highway~="" then - way:Layer("transportation", false) - way:Attribute("class", highway) --- way:Attribute("id",way:Id()) --- way:AttributeNumeric("area",37) + Layer("transportation", false) + if highway=="unclassified" or highway=="residential" then highway="minor" end + Attribute("class", highway) + -- ...and road names + local name = Find("name") + if name~="" then + Layer("transportation_name", false) + Attribute("class", highway) + Attribute("name:latin", name) + end + end + + -- Rivers + if waterway=="stream" or waterway=="river" or waterway=="canal" then + Layer("waterway", false) + Attribute("class", waterway) + AttributeNumeric("intermittent", 0) end - if waterway~="" then - way:Layer("waterway", false) - way:Attribute("class", waterway) + + -- Lakes and other water polygons + if Find("natural")=="water" then + Layer("water", true) + if Find("water")=="river" then + Attribute("class", "river") + else + Attribute("class", "lake") + end end + -- Buildings if building~="" then - way:Layer("building", true) + Layer("building", true) end end diff --git a/resources/process-openmaptiles.lua b/resources/process-openmaptiles.lua index 3e4a9981..2dd2de12 100644 --- a/resources/process-openmaptiles.lua +++ b/resources/process-openmaptiles.lua @@ -118,36 +118,36 @@ function calcRank(place, population, capital_al) end -function node_function(node) +function node_function() -- Write 'aerodrome_label' - local aeroway = node:Find("aeroway") + local aeroway = Find("aeroway") if aeroway == "aerodrome" then - node:Layer("aerodrome_label", false) - SetNameAttributes(node) - node:Attribute("iata", node:Find("iata")) - SetEleAttributes(node) - node:Attribute("icao", node:Find("icao")) + Layer("aerodrome_label", false) + SetNameAttributes() + Attribute("iata", Find("iata")) + SetEleAttributes() + Attribute("icao", Find("icao")) - local aerodrome_value = node:Find("aerodrome") + local aerodrome_value = Find("aerodrome") local class if aerodromeValues[aerodrome_value] then class = aerodrome_value else class = "other" end - node:Attribute("class", class) + Attribute("class", class) end -- Write 'housenumber' - local housenumber = node:Find("addr:housenumber") + local housenumber = Find("addr:housenumber") if housenumber~="" then - node:Layer("housenumber", false) - node:Attribute("housenumber", housenumber) + Layer("housenumber", false) + Attribute("housenumber", housenumber) end -- Write 'place' -- note that OpenMapTiles has a rank for countries (1-3), states (1-6) and cities (1-10+); -- we could potentially approximate it for cities based on the population tag - local place = node:Find("place") + local place = Find("place") if place ~= "" then local mz = 13 - local pop = tonumber(node:Find("population")) or 0 - local capital = capitalLevel(node:Find("capital")) + local pop = tonumber(Find("population")) or 0 + local capital = capitalLevel(Find("capital")) local rank = calcRank(place, pop, capital) if place == "continent" then mz=0 @@ -167,43 +167,43 @@ function node_function(node) elseif place == "locality" then mz=13 end - node:Layer("place", false) - node:Attribute("class", place) - node:MinZoom(mz) - if rank then node:AttributeNumeric("rank", rank) end - if capital then node:AttributeNumeric("capital", capital) end + Layer("place", false) + Attribute("class", place) + MinZoom(mz) + if rank then AttributeNumeric("rank", rank) end + if capital then AttributeNumeric("capital", capital) end if place=="country" then - local iso_a2 = node:Find("ISO3166-1:alpha2") + local iso_a2 = Find("ISO3166-1:alpha2") while iso_a2 == "" do - local rel, role = node:NextRelation() + local rel, role = NextRelation() if not rel then break end if role == 'label' then - iso_a2 = node:FindInRelation("ISO3166-1:alpha2") + iso_a2 = FindInRelation("ISO3166-1:alpha2") end end - node:Attribute("iso_a2", iso_a2) + Attribute("iso_a2", iso_a2) end - SetNameAttributes(node) + SetNameAttributes() return end -- Write 'poi' - local rank, class, subclass = GetPOIRank(node) - if rank then WritePOI(node,class,subclass,rank) end + local rank, class, subclass = GetPOIRank() + if rank then WritePOI(class,subclass,rank) end -- Write 'mountain_peak' and 'water_name' - local natural = node:Find("natural") + local natural = Find("natural") if natural == "peak" or natural == "volcano" then - node:Layer("mountain_peak", false) - SetEleAttributes(node) - node:AttributeNumeric("rank", 1) - node:Attribute("class", natural) - SetNameAttributes(node) + Layer("mountain_peak", false) + SetEleAttributes() + AttributeNumeric("rank", 1) + Attribute("class", natural) + SetNameAttributes() return end if natural == "bay" then - node:Layer("water_name", false) - SetNameAttributes(node) + Layer("water_name", false) + SetNameAttributes() return end end @@ -289,81 +289,81 @@ waterwayClasses = Set { "stream", "river", "canal", "drain", "ditch" } -- Scan relations for use in ways -function relation_scan_function(relation) - if relation:Find("type")=="boundary" and relation:Find("boundary")=="administrative" then - relation:Accept() +function relation_scan_function() + if Find("type")=="boundary" and Find("boundary")=="administrative" then + Accept() end end -function write_to_transportation_layer(way, minzoom, highway_class) - way:Layer("transportation", false) - way:MinZoom(minzoom) - SetZOrder(way) - way:Attribute("class", highway_class) - SetBrunnelAttributes(way) - if ramp then way:AttributeNumeric("ramp",1) end +function write_to_transportation_layer(minzoom, highway_class) + Layer("transportation", false) + MinZoom(minzoom) + SetZOrder() + Attribute("class", highway_class) + SetBrunnelAttributes() + if ramp then AttributeNumeric("ramp",1) end -- Service - if highway == "service" and service ~="" then way:Attribute("service", service) end + if highway == "service" and service ~="" then Attribute("service", service) end - local oneway = way:Find("oneway") + local oneway = Find("oneway") if oneway == "yes" or oneway == "1" then - way:AttributeNumeric("oneway",1) + AttributeNumeric("oneway",1) end if oneway == "-1" then -- **** TODO end - local surface = way:Find("surface") - local surfaceMinzoom = 12 + local surface = Find("surface") + local surfaceMinzoom = 12 if pavedValues[surface] then - way:Attribute("surface", "paved", surfaceMinzoom) + Attribute("surface", "paved", surfaceMinzoom) elseif unpavedValues[surface] then - way:Attribute("surface", "unpaved", surfaceMinzoom) - end - local accessMinzoom = 9 - if way:Holds("access") then way:Attribute("access", way:Find("access"), accessMinzoom) end - if way:Holds("bicycle") then way:Attribute("bicycle", way:Find("bicycle"), accessMinzoom) end - if way:Holds("foot") then way:Attribute("foot", way:Find("foot"), accessMinzoom) end - if way:Holds("horse") then way:Attribute("horse", way:Find("horse"), accessMinzoom) end - way:AttributeBoolean("toll", way:Find("toll") == "yes", accessMinzoom) - way:AttributeNumeric("layer", tonumber(way:Find("layer")) or 0, accessMinzoom) - way:AttributeBoolean("expressway", way:Find("expressway"), 7) - way:Attribute("mtb_scale", way:Find("mtb:scale"), 10) + Attribute("surface", "unpaved", surfaceMinzoom) + end + local accessMinzoom = 9 + if Holds("access") then Attribute("access", Find("access"), accessMinzoom) end + if Holds("bicycle") then Attribute("bicycle", Find("bicycle"), accessMinzoom) end + if Holds("foot") then Attribute("foot", Find("foot"), accessMinzoom) end + if Holds("horse") then Attribute("horse", Find("horse"), accessMinzoom) end + AttributeBoolean("toll", Find("toll") == "yes", accessMinzoom) + AttributeNumeric("layer", tonumber(Find("layer")) or 0, accessMinzoom) + AttributeBoolean("expressway", Find("expressway"), 7) + Attribute("mtb_scale", Find("mtb:scale"), 10) end -- Process way tags -function way_function(way) - local route = way:Find("route") - local highway = way:Find("highway") - local waterway = way:Find("waterway") - local water = way:Find("water") - local building = way:Find("building") - local natural = way:Find("natural") - local historic = way:Find("historic") - local landuse = way:Find("landuse") - local leisure = way:Find("leisure") - local amenity = way:Find("amenity") - local aeroway = way:Find("aeroway") - local railway = way:Find("railway") - local service = way:Find("service") - local sport = way:Find("sport") - local shop = way:Find("shop") - local tourism = way:Find("tourism") - local man_made = way:Find("man_made") - local boundary = way:Find("boundary") - local isClosed = way:IsClosed() - local housenumber = way:Find("addr:housenumber") +function way_function() + local route = Find("route") + local highway = Find("highway") + local waterway = Find("waterway") + local water = Find("water") + local building = Find("building") + local natural = Find("natural") + local historic = Find("historic") + local landuse = Find("landuse") + local leisure = Find("leisure") + local amenity = Find("amenity") + local aeroway = Find("aeroway") + local railway = Find("railway") + local service = Find("service") + local sport = Find("sport") + local shop = Find("shop") + local tourism = Find("tourism") + local man_made = Find("man_made") + local boundary = Find("boundary") + local isClosed = IsClosed() + local housenumber = Find("addr:housenumber") local write_name = false - local construction = way:Find("construction") + local construction = Find("construction") -- Miscellaneous preprocessing - if way:Find("disused") == "yes" then return end - if boundary~="" and way:Find("protection_title")=="National Forest" and way:Find("operator")=="United States Forest Service" then return end + if Find("disused") == "yes" then return end + if boundary~="" and Find("protection_title")=="National Forest" and Find("operator")=="United States Forest Service" then return end if highway == "proposed" then return end if aerowayBuildings[aeroway] then building="yes"; aeroway="" end if landuse == "field" then landuse = "farmland" end - if landuse == "meadow" and way:Find("meadow")=="agricultural" then landuse="farmland" end + if landuse == "meadow" and Find("meadow")=="agricultural" then landuse="farmland" end -- Boundaries within relations -- note that we process administrative boundaries as properties on ways, rather than as single relation geometries, @@ -371,21 +371,21 @@ function way_function(way) local admin_level = 11 local isBoundary = false while true do - local rel = way:NextRelation() + local rel = NextRelation() if not rel then break end isBoundary = true - admin_level = math.min(admin_level, tonumber(way:FindInRelation("admin_level")) or 11) + admin_level = math.min(admin_level, tonumber(FindInRelation("admin_level")) or 11) end -- Boundaries in ways if boundary=="administrative" then - admin_level = math.min(admin_level, tonumber(way:Find("admin_level")) or 11) + admin_level = math.min(admin_level, tonumber(Find("admin_level")) or 11) isBoundary = true end -- Administrative boundaries -- https://openmaptiles.org/schema/#boundary - if isBoundary and not (way:Find("maritime")=="yes") then + if isBoundary and not (Find("maritime")=="yes") then local mz = 0 if admin_level>=3 and admin_level<5 then mz=4 elseif admin_level>=5 and admin_level<7 then mz=8 @@ -393,22 +393,22 @@ function way_function(way) elseif admin_level>=8 then mz=12 end - way:Layer("boundary",false) - way:AttributeNumeric("admin_level", admin_level) - way:MinZoom(mz) + Layer("boundary",false) + AttributeNumeric("admin_level", admin_level) + MinZoom(mz) -- disputed status (0 or 1). some styles need to have the 0 to show it. - local disputed = way:Find("disputed") + local disputed = Find("disputed") if disputed=="yes" then - way:AttributeNumeric("disputed", 1) + AttributeNumeric("disputed", 1) else - way:AttributeNumeric("disputed", 0) + AttributeNumeric("disputed", 0) end end -- Roads ('transportation' and 'transportation_name', plus 'transportation_name_detail') if highway~="" then - local access = way:Find("access") - local surface = way:Find("surface") + local access = Find("access") + local surface = Find("surface") local h = highway local minzoom = 99 @@ -449,158 +449,158 @@ function way_function(way) -- Write to layer if minzoom <= 14 then - write_to_transportation_layer(way, minzoom, h) + write_to_transportation_layer(minzoom, h) -- Write names if minzoom < 8 then minzoom = 8 end if highway == "motorway" or highway == "trunk" then - way:Layer("transportation_name", false) - way:MinZoom(minzoom) + Layer("transportation_name", false) + MinZoom(minzoom) elseif h == "minor" or h == "track" or h == "path" or h == "service" then - way:Layer("transportation_name_detail", false) - way:MinZoom(minzoom) + Layer("transportation_name_detail", false) + MinZoom(minzoom) else - way:Layer("transportation_name_mid", false) - way:MinZoom(minzoom) + Layer("transportation_name_mid", false) + MinZoom(minzoom) end - SetNameAttributes(way) - way:Attribute("class",h) - way:Attribute("network","road") -- **** could also be us-interstate, us-highway, us-state - if h~=highway then way:Attribute("subclass",highway) end - local ref = way:Find("ref") + SetNameAttributes() + Attribute("class",h) + Attribute("network","road") -- **** could also be us-interstate, us-highway, us-state + if h~=highway then Attribute("subclass",highway) end + local ref = Find("ref") if ref~="" then - way:Attribute("ref",ref) - way:AttributeNumeric("ref_length",ref:len()) + Attribute("ref",ref) + AttributeNumeric("ref_length",ref:len()) end end end -- Railways ('transportation' and 'transportation_name', plus 'transportation_name_detail') if railway~="" then - way:Layer("transportation", false) - way:Attribute("class", railway) - SetZOrder(way) - SetBrunnelAttributes(way) + Layer("transportation", false) + Attribute("class", railway) + SetZOrder() + SetBrunnelAttributes() if service~="" then - way:Attribute("service", service) - way:MinZoom(12) + Attribute("service", service) + MinZoom(12) else - way:MinZoom(9) + MinZoom(9) end - way:Layer("transportation_name", false) - SetNameAttributes(way) - way:MinZoom(14) - way:Attribute("class", "rail") + Layer("transportation_name", false) + SetNameAttributes() + MinZoom(14) + Attribute("class", "rail") end -- Pier if man_made=="pier" then - way:Layer("transportation", isClosed) - SetZOrder(way) - way:Attribute("class", "pier") - SetMinZoomByArea(way) + Layer("transportation", isClosed) + SetZOrder() + Attribute("class", "pier") + SetMinZoomByArea() end -- 'Ferry' if route=="ferry" then - way:Layer("transportation", false) - way:Attribute("class", "ferry") - SetZOrder(way) - way:MinZoom(9) - SetBrunnelAttributes(way) + Layer("transportation", false) + Attribute("class", "ferry") + SetZOrder() + MinZoom(9) + SetBrunnelAttributes() - way:Layer("transportation_name", false) - SetNameAttributes(way) - way:MinZoom(12) - way:Attribute("class", "ferry") + Layer("transportation_name", false) + SetNameAttributes() + MinZoom(12) + Attribute("class", "ferry") end -- 'Aeroway' if aeroway~="" then - way:Layer("aeroway", isClosed) - way:Attribute("class",aeroway) - way:Attribute("ref",way:Find("ref")) + Layer("aeroway", isClosed) + Attribute("class",aeroway) + Attribute("ref",Find("ref")) write_name = true end -- 'aerodrome_label' if aeroway=="aerodrome" then - way:LayerAsCentroid("aerodrome_label") - SetNameAttributes(way) - way:Attribute("iata", way:Find("iata")) - SetEleAttributes(way) - way:Attribute("icao", way:Find("icao")) + LayerAsCentroid("aerodrome_label") + SetNameAttributes() + Attribute("iata", Find("iata")) + SetEleAttributes() + Attribute("icao", Find("icao")) - local aerodrome = way:Find(aeroway) + local aerodrome = Find(aeroway) local class if aerodromeValues[aerodrome] then class = aerodrome else class = "other" end - way:Attribute("class", class) + Attribute("class", class) end -- Set 'waterway' and associated if waterwayClasses[waterway] and not isClosed then - if waterway == "river" and way:Holds("name") then - way:Layer("waterway", false) + if waterway == "river" and Holds("name") then + Layer("waterway", false) else - way:Layer("waterway_detail", false) + Layer("waterway_detail", false) end - if way:Find("intermittent")=="yes" then way:AttributeNumeric("intermittent", 1) else way:AttributeNumeric("intermittent", 0) end - way:Attribute("class", waterway) - SetNameAttributes(way) - SetBrunnelAttributes(way) - elseif waterway == "boatyard" then way:Layer("landuse", isClosed); way:Attribute("class", "industrial"); way:MinZoom(12) - elseif waterway == "dam" then way:Layer("building",isClosed) - elseif waterway == "fuel" then way:Layer("landuse", isClosed); way:Attribute("class", "industrial"); way:MinZoom(14) + if Find("intermittent")=="yes" then AttributeNumeric("intermittent", 1) else AttributeNumeric("intermittent", 0) end + Attribute("class", waterway) + SetNameAttributes() + SetBrunnelAttributes() + elseif waterway == "boatyard" then Layer("landuse", isClosed); Attribute("class", "industrial"); MinZoom(12) + elseif waterway == "dam" then Layer("building",isClosed) + elseif waterway == "fuel" then Layer("landuse", isClosed); Attribute("class", "industrial"); MinZoom(14) end -- Set names on rivers if waterwayClasses[waterway] and not isClosed then - if waterway == "river" and way:Holds("name") then - way:Layer("water_name", false) + if waterway == "river" and Holds("name") then + Layer("water_name", false) else - way:Layer("water_name_detail", false) - way:MinZoom(14) + Layer("water_name_detail", false) + MinZoom(14) end - way:Attribute("class", waterway) - SetNameAttributes(way) + Attribute("class", waterway) + SetNameAttributes() end -- Set 'building' and associated if building~="" then - way:Layer("building", true) - SetBuildingHeightAttributes(way) - SetMinZoomByArea(way) + Layer("building", true) + SetBuildingHeightAttributes() + SetMinZoomByArea() end -- Set 'housenumber' if housenumber~="" then - way:LayerAsCentroid("housenumber") - way:Attribute("housenumber", housenumber) + LayerAsCentroid("housenumber") + Attribute("housenumber", housenumber) end -- Set 'water' if natural=="water" or leisure=="swimming_pool" or landuse=="reservoir" or landuse=="basin" or waterClasses[waterway] then - if way:Find("covered")=="yes" or not isClosed then return end + if Find("covered")=="yes" or not isClosed then return end local class="lake"; if waterway~="" then class="river" end - if class=="lake" and way:Find("wikidata")=="Q192770" then return end - way:Layer("water",true) + if class=="lake" and Find("wikidata")=="Q192770" then return end + Layer("water",true) SetMinZoomByArea(way) - way:Attribute("class",class) + Attribute("class",class) - if way:Find("intermittent")=="yes" then way:Attribute("intermittent",1) end + if Find("intermittent")=="yes" then Attribute("intermittent",1) end -- we only want to show the names of actual lakes not every man-made basin that probably doesn't even have a name other than "basin" -- examples for which we don't want to show a name: -- https://www.openstreetmap.org/way/25958687 -- https://www.openstreetmap.org/way/27201902 -- https://www.openstreetmap.org/way/25309134 -- https://www.openstreetmap.org/way/24579306 - if way:Holds("name") and natural=="water" and water ~= "basin" and water ~= "wastewater" then - way:LayerAsCentroid("water_name_detail") - SetNameAttributes(way) - SetMinZoomByArea(way) - way:Attribute("class", class) + if Holds("name") and natural=="water" and water ~= "basin" and water ~= "wastewater" then + LayerAsCentroid("water_name_detail") + SetNameAttributes() + SetMinZoomByArea() + Attribute("class", class) end return -- in case we get any landuse processing @@ -611,11 +611,11 @@ function way_function(way) if l=="" then l=natural end if l=="" then l=leisure end if landcoverKeys[l] then - way:Layer("landcover", true) - SetMinZoomByArea(way) - way:Attribute("class", landcoverKeys[l]) - if l=="wetland" then way:Attribute("subclass", way:Find("wetland")) - else way:Attribute("subclass", l) end + Layer("landcover", true) + SetMinZoomByArea() + Attribute("class", landcoverKeys[l]) + if l=="wetland" then Attribute("subclass", Find("wetland")) + else Attribute("subclass", l) end write_name = true -- Set 'landuse' @@ -623,31 +623,31 @@ function way_function(way) if l=="" then l=amenity end if l=="" then l=tourism end if landuseKeys[l] then - way:Layer("landuse", true) - way:Attribute("class", l) + Layer("landuse", true) + Attribute("class", l) if l=="residential" then - if way:Area()4 then layer="poi_detail" end - obj:LayerAsCentroid(layer) - SetNameAttributes(obj) - obj:AttributeNumeric("rank", rank) - obj:Attribute("class", class) - obj:Attribute("subclass", subclass) + LayerAsCentroid(layer) + SetNameAttributes() + AttributeNumeric("rank", rank) + Attribute("class", class) + Attribute("subclass", subclass) -- layer defaults to 0 - obj:AttributeNumeric("layer", tonumber(obj:Find("layer")) or 0) + AttributeNumeric("layer", tonumber(Find("layer")) or 0) -- indoor defaults to false - obj:AttributeBoolean("indoor", (obj:Find("indoor") == "yes")) + AttributeBoolean("indoor", (Find("indoor") == "yes")) -- level has no default - local level = tonumber(obj:Find("level")) + local level = tonumber(Find("level")) if level then - obj:AttributeNumeric("level", level) + AttributeNumeric("level", level) end end -- Set name attributes on any object -function SetNameAttributes(obj) - local name = obj:Find("name"), iname +function SetNameAttributes() + local name = Find("name"), iname local main_written = name -- if we have a preferred language, then write that (if available), and additionally write the base name tag - if preferred_language and obj:Holds("name:"..preferred_language) then - iname = obj:Find("name:"..preferred_language) - obj:Attribute(preferred_language_attribute, iname) + if preferred_language and Holds("name:"..preferred_language) then + iname = Find("name:"..preferred_language) + Attribute(preferred_language_attribute, iname) if iname~=name and default_language_attribute then - obj:Attribute(default_language_attribute, name) + Attribute(default_language_attribute, name) else main_written = iname end else - obj:Attribute(preferred_language_attribute, name) + Attribute(preferred_language_attribute, name) end -- then set any additional languages for i,lang in ipairs(additional_languages) do - iname = obj:Find("name:"..lang) + iname = Find("name:"..lang) if iname=="" then iname=name end - if iname~=main_written then obj:Attribute("name:"..lang, iname) end + if iname~=main_written then Attribute("name:"..lang, iname) end end end -- Set ele and ele_ft on any object -function SetEleAttributes(obj) - local ele = obj:Find("ele") +function SetEleAttributes() + local ele = Find("ele") if ele ~= "" then local meter = math.floor(tonumber(ele) or 0) local feet = math.floor(meter * 3.2808399) - obj:AttributeNumeric("ele", meter) - obj:AttributeNumeric("ele_ft", feet) + AttributeNumeric("ele", meter) + AttributeNumeric("ele_ft", feet) end end -function SetBrunnelAttributes(obj) - if obj:Find("bridge") == "yes" then obj:Attribute("brunnel", "bridge") - elseif obj:Find("tunnel") == "yes" then obj:Attribute("brunnel", "tunnel") - elseif obj:Find("ford") == "yes" then obj:Attribute("brunnel", "ford") +function SetBrunnelAttributes() + if Find("bridge") == "yes" then Attribute("brunnel", "bridge") + elseif Find("tunnel") == "yes" then Attribute("brunnel", "tunnel") + elseif Find("ford") == "yes" then Attribute("brunnel", "ford") end end -- Set minimum zoom level by area -function SetMinZoomByArea(way) - local area=way:Area() - if area>ZRES5^2 then way:MinZoom(6) - elseif area>ZRES6^2 then way:MinZoom(7) - elseif area>ZRES7^2 then way:MinZoom(8) - elseif area>ZRES8^2 then way:MinZoom(9) - elseif area>ZRES9^2 then way:MinZoom(10) - elseif area>ZRES10^2 then way:MinZoom(11) - elseif area>ZRES11^2 then way:MinZoom(12) - elseif area>ZRES12^2 then way:MinZoom(13) - else way:MinZoom(14) end +function SetMinZoomByArea() + local area=Area() + if area>ZRES5^2 then MinZoom(6) + elseif area>ZRES6^2 then MinZoom(7) + elseif area>ZRES7^2 then MinZoom(8) + elseif area>ZRES8^2 then MinZoom(9) + elseif area>ZRES9^2 then MinZoom(10) + elseif area>ZRES10^2 then MinZoom(11) + elseif area>ZRES11^2 then MinZoom(12) + elseif area>ZRES12^2 then MinZoom(13) + else MinZoom(14) end end -- Calculate POIs (typically rank 1-4 go to 'poi' z12-14, rank 5+ to 'poi_detail' z14) -- returns rank, class, subclass -function GetPOIRank(obj) +function GetPOIRank() local k,list,v,class,rank -- Can we find the tag? for k,list in pairs(poiTags) do - if list[obj:Find(k)] then - v = obj:Find(k) -- k/v are the OSM tag pair + if list[Find(k)] then + v = Find(k) -- k/v are the OSM tag pair class = poiClasses[v] or k rank = poiClassRanks[class] or 25 subclassKey = poiSubClasses[v] if subclassKey then class = v - v = obj:Find(subclassKey) + v = Find(subclassKey) end return rank, class, v end end -- Catch-all for shops - local shop = obj:Find("shop") + local shop = Find("shop") if shop~="" then return poiClassRanks['shop'], "shop", shop end -- Nothing found return nil,nil,nil end -function SetBuildingHeightAttributes(way) - local height = tonumber(way:Find("height"), 10) - local minHeight = tonumber(way:Find("min_height"), 10) - local levels = tonumber(way:Find("building:levels"), 10) - local minLevel = tonumber(way:Find("building:min_level"), 10) +function SetBuildingHeightAttributes() + local height = tonumber(Find("height"), 10) + local minHeight = tonumber(Find("min_height"), 10) + local levels = tonumber(Find("building:levels"), 10) + local minLevel = tonumber(Find("building:min_level"), 10) local renderHeight = BUILDING_FLOOR_HEIGHT if height or levels then @@ -789,17 +789,17 @@ function SetBuildingHeightAttributes(way) renderHeight = renderHeight + renderMinHeight end - way:AttributeNumeric("render_height", renderHeight) - way:AttributeNumeric("render_min_height", renderMinHeight) + AttributeNumeric("render_height", renderHeight) + AttributeNumeric("render_min_height", renderMinHeight) end -- Implement z_order as calculated by Imposm -- See https://imposm.org/docs/imposm3/latest/mapping.html#wayzorder for details. -function SetZOrder(way) - local highway = way:Find("highway") - local layer = tonumber(way:Find("layer")) - local bridge = way:Find("bridge") - local tunnel = way:Find("tunnel") +function SetZOrder() + local highway = Find("highway") + local layer = tonumber(Find("layer")) + local bridge = Find("bridge") + local tunnel = Find("tunnel") local zOrder = 0 if bridge ~= "" and bridge ~= "no" then zOrder = zOrder + 10 @@ -830,7 +830,7 @@ function SetZOrder(way) hwClass = 3 end zOrder = zOrder + hwClass - way:ZOrder(zOrder) + ZOrder(zOrder) end -- ========================================================== diff --git a/src/attribute_store.cpp b/src/attribute_store.cpp index 6fbacbe9..1f63edbe 100644 --- a/src/attribute_store.cpp +++ b/src/attribute_store.cpp @@ -73,18 +73,18 @@ const AttributePair& AttributePairStore::getPair(uint32_t i) const { if (shard == 0) { if (offset < tlsHotShard.size()) - return tlsHotShard.at(offset); + return tlsHotShard[offset]; { std::lock_guard lock(pairsMutex[0]); tlsHotShard = pairs[0]; } - return tlsHotShard.at(offset); + return tlsHotShard[offset]; } std::lock_guard lock(pairsMutex[shard]); - return pairs[shard].at(offset); + return pairs[shard][offset]; }; const AttributePair& AttributePairStore::getPairUnsafe(uint32_t i) const { @@ -94,9 +94,16 @@ const AttributePair& AttributePairStore::getPairUnsafe(uint32_t i) const { uint32_t shard = i >> (32 - SHARD_BITS); uint32_t offset = i & (~(~0u << (32 - SHARD_BITS))); - return pairs[shard].at(offset); + return pairs[shard][offset]; }; +// Remember recently queried/added pairs so that we can return them in the +// future without taking a lock. +thread_local uint64_t tlsPairLookups = 0; +thread_local uint64_t tlsPairLookupsUncached = 0; + +thread_local std::vector cachedAttributePairPointers(64); +thread_local std::vector cachedAttributePairIndexes(64); uint32_t AttributePairStore::addPair(AttributePair& pair, bool isHot) { if (isHot) { { @@ -132,6 +139,23 @@ uint32_t AttributePairStore::addPair(AttributePair& pair, bool isHot) { // Throw it on the pile with the rest of the pairs. size_t hash = pair.hash(); + const size_t candidateIndex = hash % cachedAttributePairPointers.size(); + // Before taking a lock, see if we've seen this attribute pair recently. + + tlsPairLookups++; + if (tlsPairLookups % 1024 == 0) { + lookups += 1024; + } + + + { + const AttributePair* candidate = cachedAttributePairPointers[candidateIndex]; + + if (candidate != nullptr && *candidate == pair) + return cachedAttributePairIndexes[candidateIndex]; + } + + size_t shard = hash % ATTRIBUTE_SHARDS; // Shard 0 is for hot pairs -- pick another shard if it gets selected. if (shard == 0) shard = (hash >> 8) % ATTRIBUTE_SHARDS; @@ -140,9 +164,19 @@ uint32_t AttributePairStore::addPair(AttributePair& pair, bool isHot) { if (shard == 0) shard = 1; std::lock_guard lock(pairsMutex[shard]); + + tlsPairLookupsUncached++; + if (tlsPairLookupsUncached % 1024 == 0) + lookupsUncached += 1024; + const auto& index = pairs[shard].find(pair); - if (index != -1) - return (shard << (32 - SHARD_BITS)) + index; + if (index != -1) { + const uint32_t rv = (shard << (32 - SHARD_BITS)) + index; + cachedAttributePairPointers[candidateIndex] = &pairs[shard][index]; + cachedAttributePairIndexes[candidateIndex] = rv; + + return rv; + } pair.ensureStringIsOwned(); uint32_t offset = pairs[shard].add(pair); @@ -206,23 +240,20 @@ void AttributeSet::removePairWithKey(const AttributePairStore& pairStore, uint32 } } -void AttributeStore::addAttribute(AttributeSet& attributeSet, std::string const &key, const std::string& v, char minzoom) { +void AttributeStore::addAttribute(AttributeSet& attributeSet, std::string const &key, const protozero::data_view v, char minzoom) { PooledString ps(&v); AttributePair kv(keyStore.key2index(key), ps, minzoom); bool isHot = AttributePair::isHot(key, v); - attributeSet.removePairWithKey(pairStore, kv.keyIndex); attributeSet.addPair(pairStore.addPair(kv, isHot)); } void AttributeStore::addAttribute(AttributeSet& attributeSet, std::string const &key, bool v, char minzoom) { AttributePair kv(keyStore.key2index(key),v,minzoom); bool isHot = true; // All bools are eligible to be hot pairs - attributeSet.removePairWithKey(pairStore, kv.keyIndex); attributeSet.addPair(pairStore.addPair(kv, isHot)); } void AttributeStore::addAttribute(AttributeSet& attributeSet, std::string const &key, float v, char minzoom) { AttributePair kv(keyStore.key2index(key),v,minzoom); bool isHot = v >= 0 && v <= 25 && ceil(v) == v; // Whole numbers in 0..25 are eligible to be hot pairs - attributeSet.removePairWithKey(pairStore, kv.keyIndex); attributeSet.addPair(pairStore.addPair(kv, isHot)); } @@ -263,25 +294,54 @@ void AttributeSet::finalize() { } +// Remember recently queried/added sets so that we can return them in the +// future without taking a lock. +thread_local std::vector cachedAttributeSetPointers(64); +thread_local std::vector cachedAttributeSetIndexes(64); + +thread_local uint64_t tlsSetLookups = 0; +thread_local uint64_t tlsSetLookupsUncached = 0; AttributeIndex AttributeStore::add(AttributeSet &attributes) { // TODO: there's probably a way to use C++ types to distinguish a finalized // and non-finalized AttributeSet, which would make this safer. attributes.finalize(); size_t hash = attributes.hash(); + + const size_t candidateIndex = hash % cachedAttributeSetPointers.size(); + // Before taking a lock, see if we've seen this attribute set recently. + + tlsSetLookups++; + if (tlsSetLookups % 1024 == 0) { + lookups += 1024; + } + + + { + const AttributeSet* candidate = cachedAttributeSetPointers[candidateIndex]; + + if (candidate != nullptr && *candidate == attributes) + return cachedAttributeSetIndexes[candidateIndex]; + } + size_t shard = hash % ATTRIBUTE_SHARDS; // We can't use the top 2 bits (see OutputObject's bitfields) shard = shard >> 2; std::lock_guard lock(setsMutex[shard]); - lookups++; + tlsSetLookupsUncached++; + if (tlsSetLookupsUncached % 1024 == 0) + lookupsUncached += 1024; const uint32_t offset = sets[shard].add(attributes); if (offset >= (1 << (32 - SHARD_BITS))) throw std::out_of_range("set shard overflow"); uint32_t rv = (shard << (32 - SHARD_BITS)) + offset; + + cachedAttributeSetPointers[candidateIndex] = &sets[shard][offset]; + cachedAttributeSetIndexes[candidateIndex] = rv; return rv; } @@ -317,7 +377,7 @@ size_t AttributeStore::size() const { } void AttributeStore::reportSize() const { - std::cout << "Attributes: " << size() << " sets from " << lookups.load() << " objects" << std::endl; + std::cout << "Attributes: " << size() << " sets from " << lookups.load() << " objects (" << lookupsUncached.load() << " uncached), " << pairStore.lookups.load() << " pairs (" << pairStore.lookupsUncached.load() << " uncached)" << std::endl; // Print detailed histogram of frequencies of attributes. if (false) { @@ -380,6 +440,12 @@ void AttributeStore::reset() { tlsKeys2IndexSize = 0; tlsHotShard.clear(); + + for (int i = 0; i < cachedAttributeSetPointers.size(); i++) + cachedAttributeSetPointers[i] = nullptr; + + for (int i = 0; i < cachedAttributePairPointers.size(); i++) + cachedAttributePairPointers[i] = nullptr; } void AttributeStore::finalize() { diff --git a/src/options_parser.cpp b/src/options_parser.cpp index 529e5f4a..e13cb80e 100644 --- a/src/options_parser.cpp +++ b/src/options_parser.cpp @@ -35,7 +35,6 @@ po::options_description getParser(OptionsParser::Options& options) { ("compact",po::bool_switch(&options.osm.compact), "use faster data structure for node lookups\nNOTE: This requires the input to be renumbered (osmium renumber)") ("no-compress-nodes", po::bool_switch(&options.osm.uncompressedNodes), "store nodes uncompressed") ("no-compress-ways", po::bool_switch(&options.osm.uncompressedWays), "store ways uncompressed") - ("lazy-geometries", po::bool_switch(&options.osm.lazyGeometries), "generate geometries from the OSM stores; uses less memory") ("materialize-geometries", po::bool_switch(&options.osm.materializeGeometries), "materialize geometries; uses more memory") ("shard-stores", po::bool_switch(&options.osm.shardStores), "use an alternate reading/writing strategy for low-memory machines") ("threads",po::value(&options.threadNum)->default_value(0), "number of threads (automatically detected if 0)") @@ -67,19 +66,19 @@ OptionsParser::Options OptionsParser::parse(const int argc, const char* argv[]) po::notify(vm); if (options.osm.storeFile.empty()) { - options.osm.materializeGeometries = true; + // we're running in memory (no --store) + // so we default to lazy geometries, but --fast switches to materialized + if (options.osm.fast) { + options.osm.materializeGeometries = true; + } } else { + // we're running on-disk (--store) + // so we default to sharded, but --fast switches to unsharded if (!options.osm.fast) { options.osm.shardStores = true; } } - // You can pass --lazy-geometries to override the default of materialized geometries for - // the non-store case. - if (options.osm.lazyGeometries) - options.osm.materializeGeometries = false; - - if (vm.count("help")) { options.showHelp = true; return options; diff --git a/src/osm_lua_processing.cpp b/src/osm_lua_processing.cpp index 2a7995fe..2a50ff53 100644 --- a/src/osm_lua_processing.cpp +++ b/src/osm_lua_processing.cpp @@ -5,14 +5,159 @@ #include "helpers.h" #include "coordinates_geom.h" #include "osm_mem_tiles.h" +#include "significant_tags.h" +#include "tag_map.h" #include "node_store.h" #include "polylabel.h" +#include using namespace std; +const std::string EMPTY_STRING = ""; thread_local kaguya::State *g_luaState = nullptr; +thread_local OsmLuaProcessing* osmLuaProcessing = nullptr; + +void handleOsmLuaProcessingUserSignal(int signum) { + osmLuaProcessing->handleUserSignal(signum); +} + +class Sigusr1Handler { +public: + Sigusr1Handler() { +#ifndef _WIN32 + signal(SIGUSR1, handleOsmLuaProcessingUserSignal); +#endif + } + + void initialize() { + // No-op just to ensure the compiler doesn't optimize away + // the handler. + } +}; + +thread_local Sigusr1Handler sigusr1Handler; + +// A key in `currentTags`. If Lua code refers to an absent key, +// found will be false. +struct KnownTagKey { + bool found; + uint32_t index; + + // stringValue is populated only in PostScanRelations phase; we could consider + // having osm_store's relationTags use TagMap, in which case we'd be able to + // use the found/index fields + std::string stringValue; +}; + +template<> struct kaguya::lua_type_traits { + typedef KnownTagKey get_type; + typedef const KnownTagKey& push_type; + + static bool strictCheckType(lua_State* l, int index) + { + return lua_type(l, index) == LUA_TSTRING; + } + static bool checkType(lua_State* l, int index) + { + return lua_isstring(l, index) != 0; + } + static get_type get(lua_State* l, int index) + { + KnownTagKey rv = { false, 0 }; + size_t size = 0; + const char* buffer = lua_tolstring(l, index, &size); + + if (osmLuaProcessing->isPostScanRelation) { + // In this phase, the Holds/Find functions directly query a + // traditional string->string map, so just ensure we expose + // the string. + rv.stringValue = std::string(buffer, size); + return rv; + } + + + int64_t tagLoc = osmLuaProcessing->currentTags->getKey(buffer, size); + + if (tagLoc >= 0) { + rv.found = true; + rv.index = tagLoc; + } +// std::string key(buffer, size); +// std::cout << "for key " << key << ": rv.found=" << rv.found << ", rv.index=" << rv.index << std::endl; + return rv; + } + static int push(lua_State* l, push_type s) + { + throw std::runtime_error("Lua code doesn't know how to use KnownTagKey"); + } +}; + +template<> struct kaguya::lua_type_traits { + typedef protozero::data_view get_type; + typedef const protozero::data_view& push_type; + + static bool strictCheckType(lua_State* l, int index) + { + return lua_type(l, index) == LUA_TSTRING; + } + static bool checkType(lua_State* l, int index) + { + return lua_isstring(l, index) != 0; + } + static get_type get(lua_State* l, int index) + { + size_t size = 0; + const char* buffer = lua_tolstring(l, index, &size); + protozero::data_view rv = { buffer, size }; + return rv; + } + static int push(lua_State* l, push_type s) + { + throw std::runtime_error("Lua code doesn't know how to use protozero::data_view"); + } +}; + +std::string rawId() { return osmLuaProcessing->Id(); } +bool rawHolds(const KnownTagKey& key) { + if (osmLuaProcessing->isPostScanRelation) { + return osmLuaProcessing->Holds(key.stringValue); + } + + return key.found; +} +bool rawHasTags() { return osmLuaProcessing->HasTags(); } +void rawSetTag(const std::string &key, const std::string &value) { return osmLuaProcessing->SetTag(key, value); } +const std::string rawFind(const KnownTagKey& key) { + if (osmLuaProcessing->isPostScanRelation) + return osmLuaProcessing->Find(key.stringValue); + + if (key.found) { + auto value = *(osmLuaProcessing->currentTags->getValueFromKey(key.index)); + return std::string(value.data(), value.size()); + } + + return EMPTY_STRING; +} +std::vector rawFindIntersecting(const std::string &layerName) { return osmLuaProcessing->FindIntersecting(layerName); } +bool rawIntersects(const std::string& layerName) { return osmLuaProcessing->Intersects(layerName); } +std::vector rawFindCovering(const std::string& layerName) { return osmLuaProcessing->FindCovering(layerName); } +bool rawCoveredBy(const std::string& layerName) { return osmLuaProcessing->CoveredBy(layerName); } +bool rawIsClosed() { return osmLuaProcessing->IsClosed(); } +double rawArea() { return osmLuaProcessing->Area(); } +double rawLength() { return osmLuaProcessing->Length(); } +std::vector rawCentroid(kaguya::VariadicArgType algorithm) { return osmLuaProcessing->Centroid(algorithm); } +void rawLayer(const std::string& layerName, bool area) { return osmLuaProcessing->Layer(layerName, area); } +void rawLayerAsCentroid(const std::string &layerName, kaguya::VariadicArgType nodeSources) { return osmLuaProcessing->LayerAsCentroid(layerName, nodeSources); } +void rawMinZoom(const double z) { return osmLuaProcessing->MinZoom(z); } +void rawZOrder(const double z) { return osmLuaProcessing->ZOrder(z); } +OsmLuaProcessing::OptionalRelation rawNextRelation() { return osmLuaProcessing->NextRelation(); } +void rawRestartRelations() { return osmLuaProcessing->RestartRelations(); } +std::string rawFindInRelation(const std::string& key) { return osmLuaProcessing->FindInRelation(key); } +void rawAccept() { return osmLuaProcessing->Accept(); } +double rawAreaIntersecting(const std::string& layerName) { return osmLuaProcessing->AreaIntersecting(layerName); } + + bool supportsRemappingShapefiles = false; -const std::string EMPTY_STRING = ""; int lua_error_handler(int errCode, const char *errMessage) { @@ -42,37 +187,52 @@ OsmLuaProcessing::OsmLuaProcessing( layers(layers), materializeGeometries(materializeGeometries) { + sigusr1Handler.initialize(); + // ---- Initialise Lua g_luaState = &luaState; luaState.setErrorHandler(lua_error_handler); luaState.dofile(luaFile.c_str()); - luaState["OSM"].setClass(kaguya::UserdataMetatable() - .addFunction("Id", &OsmLuaProcessing::Id) - .addFunction("Holds", &OsmLuaProcessing::Holds) - .addFunction("Find", &OsmLuaProcessing::Find) - .addFunction("FindIntersecting", &OsmLuaProcessing::FindIntersecting) - .addFunction("Intersects", &OsmLuaProcessing::Intersects) - .addFunction("FindCovering", &OsmLuaProcessing::FindCovering) - .addFunction("CoveredBy", &OsmLuaProcessing::CoveredBy) - .addFunction("IsClosed", &OsmLuaProcessing::IsClosed) - .addFunction("Area", &OsmLuaProcessing::Area) - .addFunction("AreaIntersecting", &OsmLuaProcessing::AreaIntersecting) - .addFunction("Length", &OsmLuaProcessing::Length) - .addFunction("Centroid", &OsmLuaProcessing::Centroid) - .addFunction("Layer", &OsmLuaProcessing::Layer) - .addFunction("LayerAsCentroid", &OsmLuaProcessing::LayerAsCentroid) - .addOverloadedFunctions("Attribute", &OsmLuaProcessing::Attribute, &OsmLuaProcessing::AttributeWithMinZoom) - .addOverloadedFunctions("AttributeNumeric", &OsmLuaProcessing::AttributeNumeric, &OsmLuaProcessing::AttributeNumericWithMinZoom) - .addOverloadedFunctions("AttributeBoolean", &OsmLuaProcessing::AttributeBoolean, &OsmLuaProcessing::AttributeBooleanWithMinZoom) - .addFunction("MinZoom", &OsmLuaProcessing::MinZoom) - .addFunction("ZOrder", &OsmLuaProcessing::ZOrder) - .addFunction("Accept", &OsmLuaProcessing::Accept) - .addFunction("NextRelation", &OsmLuaProcessing::NextRelation) - .addFunction("RestartRelations", &OsmLuaProcessing::RestartRelations) - .addFunction("FindInRelation", &OsmLuaProcessing::FindInRelation) + + osmLuaProcessing = this; + luaState["Id"] = &rawId; + luaState["Holds"] = &rawHolds; + luaState["Find"] = &rawFind; + luaState["HasTags"] = &rawHasTags; + luaState["SetTag"] = &rawSetTag; + luaState["FindIntersecting"] = &rawFindIntersecting; + luaState["Intersects"] = &rawIntersects; + luaState["FindCovering"] = &rawFindCovering; + luaState["CoveredBy"] = &rawCoveredBy; + luaState["IsClosed"] = &rawIsClosed; + luaState["Area"] = &rawArea; + luaState["AreaIntersecting"] = &rawAreaIntersecting; + luaState["Length"] = &rawLength; + luaState["Centroid"] = &rawCentroid; + luaState["Layer"] = &rawLayer; + luaState["LayerAsCentroid"] = &rawLayerAsCentroid; + luaState["Attribute"] = kaguya::overload( + [](const std::string &key, const protozero::data_view val) { osmLuaProcessing->Attribute(key, val, 0); }, + [](const std::string &key, const protozero::data_view val, const char minzoom) { osmLuaProcessing->Attribute(key, val, minzoom); } + ); + luaState["AttributeNumeric"] = kaguya::overload( + [](const std::string &key, const float val) { osmLuaProcessing->AttributeNumeric(key, val, 0); }, + [](const std::string &key, const float val, const char minzoom) { osmLuaProcessing->AttributeNumeric(key, val, minzoom); } + ); + luaState["AttributeBoolean"] = kaguya::overload( + [](const std::string &key, const bool val) { osmLuaProcessing->AttributeBoolean(key, val, 0); }, + [](const std::string &key, const bool val, const char minzoom) { osmLuaProcessing->AttributeBoolean(key, val, minzoom); } ); + + luaState["MinZoom"] = &rawMinZoom; + luaState["ZOrder"] = &rawZOrder; + luaState["Accept"] = &rawAccept; + luaState["NextRelation"] = &rawNextRelation; + luaState["RestartRelations"] = &rawRestartRelations; + luaState["FindInRelation"] = &rawFindInRelation; supportsRemappingShapefiles = !!luaState["attribute_function"]; supportsReadingRelations = !!luaState["relation_scan_function"]; + supportsPostScanRelations = !!luaState["relation_postscan_function"]; supportsWritingRelations = !!luaState["relation_function"]; // ---- Call init_function of Lua logic @@ -87,6 +247,10 @@ OsmLuaProcessing::~OsmLuaProcessing() { luaState("if exit_function~=nil then exit_function() end"); } +void OsmLuaProcessing::handleUserSignal(int signum) { + std::cout << "processing OSM ID " << originalOsmID << std::endl; +} + // ---- Helpers provided for main routine // Has this object been assigned to any layers? @@ -102,6 +266,10 @@ bool OsmLuaProcessing::canReadRelations() { return supportsReadingRelations; } +bool OsmLuaProcessing::canPostScanRelations() { + return supportsPostScanRelations; +} + bool OsmLuaProcessing::canWriteRelations() { return supportsWritingRelations; } @@ -124,14 +292,21 @@ string OsmLuaProcessing::Id() const { // Check if there's a value for a given key bool OsmLuaProcessing::Holds(const string& key) const { - return currentTags->find(key) != currentTags->end(); + // NOTE: this is only called in the PostScanRelation phase -- other phases are handled in rawHolds + return currentPostScanTags->find(key)!=currentPostScanTags->end(); } // Get an OSM tag for a given key (or return empty string if none) const string OsmLuaProcessing::Find(const string& key) const { - auto it = currentTags->find(key); - if(it == currentTags->end()) return EMPTY_STRING; - return std::string(it->second.data(), it->second.size()); + // NOTE: this is only called in the PostScanRelation phase -- other phases are handled in rawFind + auto it = currentPostScanTags->find(key); + if (it == currentPostScanTags->end()) return EMPTY_STRING; + return it->second; +} + +// Check if an object has any tags +bool OsmLuaProcessing::HasTags() const { + return isPostScanRelation ? !currentPostScanTags->empty() : !currentTags->empty(); } // ---- Spatial queries called from Lua @@ -282,7 +457,7 @@ double OsmLuaProcessing::multiPolygonArea(const MultiPolygon &mp) const { // Returns length double OsmLuaProcessing::Length() { - if (isWay) { + if (isWay && !isRelation) { geom::model::linestring l; geom::assign(l, linestringCached()); geom::for_each_point(l, reverse_project); @@ -331,6 +506,7 @@ const MultiPolygon &OsmLuaProcessing::multiPolygonCached() { // Add object to specified layer from Lua void OsmLuaProcessing::Layer(const string &layerName, bool area) { + outputKeys.clear(); if (layers.layerMap.count(layerName) == 0) { throw out_of_range("ERROR: Layer(): a layer named as \"" + layerName + "\" doesn't exist."); } @@ -456,6 +632,7 @@ void OsmLuaProcessing::Layer(const string &layerName, bool area) { // When called for a relation, you can pass a list of roles. The point of a node // with that role will be used if available. void OsmLuaProcessing::LayerAsCentroid(const string &layerName, kaguya::VariadicArgType varargs) { + outputKeys.clear(); if (layers.layerMap.count(layerName) == 0) { throw out_of_range("ERROR: LayerAsCentroid(): a layer named as \"" + layerName + "\" doesn't exist."); } @@ -480,11 +657,11 @@ void OsmLuaProcessing::LayerAsCentroid(const string &layerName, kaguya::Variadic // If we're a relation, see if the user would prefer we use one of its members // to label the point. if (isRelation) { - size_t i = 0; + int i = -1; for (auto needleRef : varargs) { + i++; // Skip the first vararg, it's the algorithm. if (i == 0) continue; - i++; const std::string needle = needleRef.get(); // We do a linear search of the relation's members. This is not very efficient @@ -627,26 +804,44 @@ std::vector OsmLuaProcessing::Centroid(kaguya::VariadicArgType algorithm void OsmLuaProcessing::Accept() { relationAccepted = true; } +// Set a tag in post-scan phase +void OsmLuaProcessing::SetTag(const std::string &key, const std::string &value) { + if (!isPostScanRelation) throw std::runtime_error("SetTag can only be used in relation_postscan_function"); + osmStore.scannedRelations.set_relation_tag(originalOsmID, key, value); +} + +void OsmLuaProcessing::removeAttributeIfNeeded(const string& key) { + // Does it exist? + for (int i = 0; i < outputKeys.size(); i++) { + if (outputKeys[i] == key) { + AttributeSet& set = outputs.back().second; + set.removePairWithKey(attributeStore.pairStore, attributeStore.keyStore.key2index(key)); + return; + } + } + + outputKeys.push_back(key); +} // Set attributes in a vector tile's Attributes table -void OsmLuaProcessing::Attribute(const string &key, const string &val) { AttributeWithMinZoom(key,val,0); } -void OsmLuaProcessing::AttributeWithMinZoom(const string &key, const string &val, const char minzoom) { +void OsmLuaProcessing::Attribute(const string &key, const protozero::data_view val, const char minzoom) { if (val.size()==0) { return; } // don't set empty strings if (outputs.size()==0) { ProcessingError("Can't add Attribute if no Layer set"); return; } + removeAttributeIfNeeded(key); attributeStore.addAttribute(outputs.back().second, key, val, minzoom); setVectorLayerMetadata(outputs.back().first.layer, key, 0); } -void OsmLuaProcessing::AttributeNumeric(const string &key, const float val) { AttributeNumericWithMinZoom(key,val,0); } -void OsmLuaProcessing::AttributeNumericWithMinZoom(const string &key, const float val, const char minzoom) { +void OsmLuaProcessing::AttributeNumeric(const string &key, const float val, const char minzoom) { if (outputs.size()==0) { ProcessingError("Can't add Attribute if no Layer set"); return; } + removeAttributeIfNeeded(key); attributeStore.addAttribute(outputs.back().second, key, val, minzoom); setVectorLayerMetadata(outputs.back().first.layer, key, 1); } -void OsmLuaProcessing::AttributeBoolean(const string &key, const bool val) { AttributeBooleanWithMinZoom(key,val,0); } -void OsmLuaProcessing::AttributeBooleanWithMinZoom(const string &key, const bool val, const char minzoom) { +void OsmLuaProcessing::AttributeBoolean(const string &key, const bool val, const char minzoom) { if (outputs.size()==0) { ProcessingError("Can't add Attribute if no Layer set"); return; } + removeAttributeIfNeeded(key); attributeStore.addAttribute(outputs.back().second, key, val, minzoom); setVectorLayerMetadata(outputs.back().first.layer, key, 2); } @@ -719,34 +914,43 @@ void OsmLuaProcessing::setVectorLayerMetadata(const uint_least8_t layer, const s // Scan relation (but don't write geometry) // return true if we want it, false if we don't -bool OsmLuaProcessing::scanRelation(WayID id, const tag_map_t &tags) { +bool OsmLuaProcessing::scanRelation(WayID id, const TagMap& tags) { reset(); originalOsmID = id; - isWay = false; isRelation = true; currentTags = &tags; try { - luaState["relation_scan_function"](this); + luaState["relation_scan_function"](); } catch(luaProcessingException &e) { std::cerr << "Lua error on scanning relation " << originalOsmID << std::endl; exit(1); } if (!relationAccepted) return false; - boost::container::flat_map m; - for (const auto& i : tags) { - m[std::string(i.first.data(), i.first.size())] = std::string(i.second.data(), i.second.size()); - } - osmStore.scannedRelations.store_relation_tags(id, m); + // If we're persisting, we need to make a real map that owns its + // own keys and values. + osmStore.scannedRelations.store_relation_tags(id, tags.exportToBoostMap()); return true; } -void OsmLuaProcessing::setNode(NodeID id, LatpLon node, const tag_map_t &tags) { +// Post-scan relations - typically used for bouncing down values from nested relations +void OsmLuaProcessing::postScanRelations() { + if (!supportsPostScanRelations) return; + + for (const auto &relp : osmStore.scannedRelations.relationsForRelations) { + reset(); + isPostScanRelation = true; + RelationID id = relp.first; + originalOsmID = id; + currentPostScanTags = &(osmStore.scannedRelations.relation_tags(id)); + relationList = osmStore.scannedRelations.relations_for_relation_with_parents(id); + luaState["relation_postscan_function"](this); + } +} +bool OsmLuaProcessing::setNode(NodeID id, LatpLon node, const TagMap& tags) { reset(); originalOsmID = id; - isWay = false; - isRelation = false; lon = node.lon; latp= node.latp; currentTags = &tags; @@ -757,7 +961,7 @@ void OsmLuaProcessing::setNode(NodeID id, LatpLon node, const tag_map_t &tags) { //Start Lua processing for node try { - luaState["node_function"](this); + luaState["node_function"](); } catch(luaProcessingException &e) { std::cerr << "Lua error on node " << originalOsmID << std::endl; exit(1); @@ -769,16 +973,19 @@ void OsmLuaProcessing::setNode(NodeID id, LatpLon node, const tag_map_t &tags) { for (auto &output : finalizeOutputs()) { osmMemTiles.addObjectToSmallIndex(index, output, originalOsmID); } - } + + return true; + } + + return false; } // We are now processing a way -bool OsmLuaProcessing::setWay(WayID wayId, LatpLonVec const &llVec, const tag_map_t &tags) { +bool OsmLuaProcessing::setWay(WayID wayId, LatpLonVec const &llVec, const TagMap& tags) { reset(); wayEmitted = false; originalOsmID = wayId; isWay = true; - isRelation = false; llVecPtr = &llVec; outerWayVecPtr = nullptr; innerWayVecPtr = nullptr; @@ -804,7 +1011,7 @@ bool OsmLuaProcessing::setWay(WayID wayId, LatpLonVec const &llVec, const tag_ma //Start Lua processing for way try { kaguya::LuaFunction way_function = luaState["way_function"]; - kaguya::LuaRef ret = way_function(this); + kaguya::LuaRef ret = way_function(); assert(!ret); } catch(luaProcessingException &e) { std::cerr << "Lua error on way " << originalOsmID << std::endl; @@ -826,7 +1033,7 @@ void OsmLuaProcessing::setRelation( const PbfReader::Relation& relation, const WayVec& outerWayVec, const WayVec& innerWayVec, - const tag_map_t &tags, + const TagMap& tags, bool isNativeMP, // only OSM type=multipolygon bool isInnerOuter // any OSM relation with "inner" and "outer" roles (e.g. type=multipolygon|boundary) ) { @@ -843,10 +1050,14 @@ void OsmLuaProcessing::setRelation( innerWayVecPtr = &innerWayVec; currentTags = &tags; + if (supportsReadingRelations && osmStore.scannedRelations.relation_in_any_relations(originalOsmID)) { + relationList = osmStore.scannedRelations.relations_for_relation(originalOsmID); + } + // Start Lua processing for relation if (!isNativeMP && !supportsWritingRelations) return; try { - luaState[isNativeMP ? "way_function" : "relation_function"](this); + luaState[isNativeMP ? "way_function" : "relation_function"](); } catch(luaProcessingException &e) { std::cerr << "Lua error on relation " << originalOsmID << std::endl; exit(1); @@ -865,10 +1076,25 @@ void OsmLuaProcessing::setRelation( } } -vector OsmLuaProcessing::GetSignificantNodeKeys() { - return luaState["node_keys"]; +SignificantTags OsmLuaProcessing::GetSignificantNodeKeys() { + if (!!luaState["node_keys"]) { + std::vector keys = luaState["node_keys"]; + return SignificantTags(keys); + } + + return SignificantTags(); } +SignificantTags OsmLuaProcessing::GetSignificantWayKeys() { + if (!!luaState["way_keys"]) { + std::vector keys = luaState["way_keys"]; + return SignificantTags(keys); + } + + return SignificantTags(); +} + + std::vector OsmLuaProcessing::finalizeOutputs() { std::vector list; list.reserve(this->outputs.size()); diff --git a/src/osm_store.cpp b/src/osm_store.cpp index 98449917..5b7366b7 100644 --- a/src/osm_store.cpp +++ b/src/osm_store.cpp @@ -17,6 +17,44 @@ static inline bool isClosed(const std::vector& way) { return way.begin() == way.end(); } +UsedObjects::UsedObjects(Status status): status(status), mutex(256), ids(256 * 1024) { +} + +bool UsedObjects::test(NodeID id) { + if (status == Status::Disabled) + return true; + + const size_t chunk = id / 65536; + if (ids[chunk].size() == 0) + return false; + + return ids[chunk][id % 65536]; +} + +void UsedObjects::enable() { + status = Status::Enabled; +} + +bool UsedObjects::enabled() const { + return status == Status::Enabled; +} + +void UsedObjects::set(NodeID id) { + const size_t chunk = id / 65536; + + std::lock_guard lock(mutex[chunk % mutex.size()]); + if (ids[chunk].size() == 0) + ids[chunk].resize(65536); + + ids[chunk][id % 65536] = true; +} + +void UsedObjects::clear() { + // This data is not needed after PbfProcessor's ReadPhase::Nodes has completed, + // and it takes up to ~1.5GB of RAM. + ids.clear(); +} + void OSMStore::open(std::string const &osm_store_filename) { void_mmap_allocator::openMmapFile(osm_store_filename); diff --git a/src/pbf_processor.cpp b/src/pbf_processor.cpp index b89c3d0b..a378d297 100644 --- a/src/pbf_processor.cpp +++ b/src/pbf_processor.cpp @@ -24,41 +24,35 @@ PbfProcessor::PbfProcessor(OSMStore &osmStore) : osmStore(osmStore) { } -bool PbfProcessor::ReadNodes(OsmLuaProcessing& output, PbfReader::PrimitiveGroup& pg, const PbfReader::PrimitiveBlock& pb, const unordered_set& nodeKeyPositions) +bool PbfProcessor::ReadNodes(OsmLuaProcessing& output, PbfReader::PrimitiveGroup& pg, const PbfReader::PrimitiveBlock& pb, const SignificantTags& nodeKeys) { // ---- Read nodes std::vector nodes; + TagMap tags; + for (auto& node : pg.nodes()) { NodeID nodeId = node.id; LatpLon latplon = { int(lat2latp(double(node.lat)/10000000.0)*10000000.0), node.lon }; - bool significant = false; - for (int i = node.tagStart; i < node.tagEnd; i += 2) { - auto keyIndex = pg.translateNodeKeyValue(i); + tags.reset(); + // For tagged nodes, call Lua, then save the OutputObject + for (int n = node.tagStart; n < node.tagEnd; n += 2) { + auto keyIndex = pg.translateNodeKeyValue(n); + auto valueIndex = pg.translateNodeKeyValue(n + 1); - if (nodeKeyPositions.find(keyIndex) != nodeKeyPositions.end()) { - significant = true; - } + const protozero::data_view& key = pb.stringTable[keyIndex]; + const protozero::data_view& value = pb.stringTable[valueIndex]; + tags.addTag(key, value); } - nodes.push_back(std::make_pair(static_cast(nodeId), latplon)); - - if (significant) { - // For tagged nodes, call Lua, then save the OutputObject - boost::container::flat_map tags; - tags.reserve((node.tagEnd - node.tagStart) / 2); - - for (int n = node.tagStart; n < node.tagEnd; n += 2) { - auto keyIndex = pg.translateNodeKeyValue(n); - auto valueIndex = pg.translateNodeKeyValue(n + 1); + bool emitted = false; + if (!tags.empty() && nodeKeys.filter(tags)) { + emitted = output.setNode(static_cast(nodeId), latplon, tags); + } - protozero::data_view key{pb.stringTable[keyIndex].data(), pb.stringTable[keyIndex].size()}; - protozero::data_view value{pb.stringTable[valueIndex].data(), pb.stringTable[valueIndex].size()}; - tags[key] = value; - } - output.setNode(static_cast(nodeId), latplon, tags); - } + if (emitted || osmStore.usedNodes.test(nodeId)) + nodes.push_back(std::make_pair(static_cast(nodeId), latplon)); } if (nodes.size() > 0) { @@ -72,6 +66,7 @@ bool PbfProcessor::ReadWays( OsmLuaProcessing &output, PbfReader::PrimitiveGroup& pg, const PbfReader::PrimitiveBlock& pb, + const SignificantTags& wayKeys, bool locationsOnWays, uint shard, uint effectiveShards @@ -84,10 +79,17 @@ bool PbfProcessor::ReadWays( std::vector llWays; std::vector>> nodeWays; + TagMap tags; LatpLonVec llVec; std::vector nodeVec; for (PbfReader::Way pbfWay : pg.ways()) { + tags.reset(); + readTags(pbfWay, pb, tags); + + if (!osmStore.way_is_used(pbfWay.id) && !wayKeys.filter(tags)) + continue; + llVec.clear(); nodeVec.clear(); @@ -131,8 +133,6 @@ bool PbfProcessor::ReadWays( if (llVec.empty()) continue; try { - tag_map_t tags; - readTags(pbfWay, pb, tags); bool emitted = output.setWay(static_cast(pbfWay.id), llVec, tags); // If we need it for later, store the way's coordinates in the global way store @@ -159,11 +159,38 @@ bool PbfProcessor::ReadWays( return true; } -bool PbfProcessor::ScanRelations(OsmLuaProcessing& output, PbfReader::PrimitiveGroup& pg, const PbfReader::PrimitiveBlock& pb) { +bool PbfProcessor::ScanWays(OsmLuaProcessing& output, PbfReader::PrimitiveGroup& pg, const PbfReader::PrimitiveBlock& pb, const SignificantTags& wayKeys) { + // Scan ways to see which nodes we need to save. + // + // This phase only runs if the Lua script has declared a `way_keys` variable. + if (pg.ways().empty()) + return false; + + TagMap tags; + + // Note: unlike ScanRelations, we don't call into Lua. Instead, we statically inspect + // the tags on each way to decide if it will be emitted. + for (auto& way : pg.ways()) { + tags.reset(); + readTags(way, pb, tags); + + if (osmStore.way_is_used(way.id) || wayKeys.filter(tags)) { + for (const auto id : way.refs) { + osmStore.usedNodes.set(id); + } + } + } + + return true; +} + +bool PbfProcessor::ScanRelations(OsmLuaProcessing& output, PbfReader::PrimitiveGroup& pg, const PbfReader::PrimitiveBlock& pb, const SignificantTags& wayKeys) { // Scan relations to see which ways we need to save if (pg.relations().empty()) return false; + TagMap tags; + int typeKey = findStringPosition(pb, "type"); int mpKey = findStringPosition(pb, "multipolygon"); @@ -171,14 +198,20 @@ bool PbfProcessor::ScanRelations(OsmLuaProcessing& output, PbfReader::PrimitiveG bool isMultiPolygon = relationIsType(pbfRelation, typeKey, mpKey); bool isAccepted = false; WayID relid = static_cast(pbfRelation.id); + tags.reset(); + readTags(pbfRelation, pb, tags); + if (!isMultiPolygon) { if (output.canReadRelations()) { - tag_map_t tags; - readTags(pbfRelation, pb, tags); isAccepted = output.scanRelation(relid, tags); } + if (!isAccepted) continue; + } else { + if (!wayKeys.filter(tags)) + continue; } + osmStore.usedRelations.set(relid); for (int n=0; n < pbfRelation.memids.size(); n++) { uint64_t lastID = pbfRelation.memids[n]; @@ -187,6 +220,15 @@ bool PbfProcessor::ScanRelations(OsmLuaProcessing& output, PbfReader::PrimitiveG const auto& roleView = pb.stringTable[pbfRelation.roles_sid[n]]; std::string role(roleView.data(), roleView.size()); osmStore.scannedRelations.relation_contains_node(relid, lastID, role); + + if (osmStore.usedNodes.enabled()) + osmStore.usedNodes.set(lastID); + } + } else if (pbfRelation.types[n] == PbfReader::Relation::MemberType::RELATION) { + if (isAccepted) { + const auto& roleView = pb.stringTable[pbfRelation.roles_sid[n]]; + std::string role(roleView.data(), roleView.size()); + osmStore.scannedRelations.relation_contains_relation(relid, lastID, role); } } else if (pbfRelation.types[n] == PbfReader::Relation::MemberType::WAY) { if (lastID >= pow(2,42)) throw std::runtime_error("Way ID in relation "+std::to_string(relid)+" negative or too large: "+std::to_string(lastID)); @@ -207,6 +249,7 @@ bool PbfProcessor::ReadRelations( PbfReader::PrimitiveGroup& pg, const PbfReader::PrimitiveBlock& pb, const BlockMetadata& blockMetadata, + const SignificantTags& wayKeys, uint shard, uint effectiveShards ) { @@ -214,6 +257,7 @@ bool PbfProcessor::ReadRelations( if (pg.relations().empty()) return false; + TagMap tags; std::vector relations; int typeKey = findStringPosition(pb, "type"); @@ -257,9 +301,23 @@ bool PbfProcessor::ReadRelations( continue; try { - tag_map_t tags; - readTags(pbfRelation, pb, tags); - output.setRelation(pb.stringTable, pbfRelation, outerWayVec, innerWayVec, tags, isMultiPolygon, isInnerOuter); + tags.reset(); + std::deque dataviews; + if (osmStore.scannedRelations.has_relation_tags(pbfRelation.id)) { + const auto& scannedTags = osmStore.scannedRelations.relation_tags(pbfRelation.id); + for (const auto& entry : scannedTags) { + dataviews.push_back({entry.first.data(), entry.first.size()}); + const auto& key = dataviews.back(); + dataviews.push_back({entry.second.data(), entry.second.size()}); + const auto& value = dataviews.back(); + tags.addTag(key, value); + } + } else { + readTags(pbfRelation, pb, tags); + } + + if (osmStore.usedRelations.test(pbfRelation.id) || wayKeys.filter(tags)) + output.setRelation(pb.stringTable, pbfRelation, outerWayVec, innerWayVec, tags, isMultiPolygon, isInnerOuter); } catch (std::out_of_range &err) { // Relation is missing a member? @@ -277,7 +335,8 @@ bool PbfProcessor::ReadBlock( std::istream& infile, OsmLuaProcessing& output, const BlockMetadata& blockMetadata, - const unordered_set& nodeKeys, + const SignificantTags& nodeKeys, + const SignificantTags& wayKeys, bool locationsOnWays, ReadPhase phase, uint shard, @@ -295,14 +354,6 @@ bool PbfProcessor::ReadBlock( // Keep count of groups read during this phase. std::size_t read_groups = 0; - // Read the string table, and pre-calculate the positions of valid node keys - unordered_set nodeKeyPositions; - for (auto it : nodeKeys) { - //nodeKeyPositions.insert(findStringPosition(pb, it)); - auto rv = findStringPosition(pb, it); - nodeKeyPositions.insert(rv); - } - int primitiveGroupSize = 0; for (auto& pg : pb.groups()) { primitiveGroupSize++; @@ -325,7 +376,7 @@ bool PbfProcessor::ReadBlock( }; if(phase == ReadPhase::Nodes) { - bool done = ReadNodes(output, pg, pb, nodeKeyPositions); + bool done = ReadNodes(output, pg, pb, nodeKeys); if(done) { output_progress(); ++read_groups; @@ -333,9 +384,21 @@ bool PbfProcessor::ReadBlock( } } + if(phase == ReadPhase::WayScan) { + bool done = ScanWays(output, pg, pb, wayKeys); + if(done) { + if (ioMutex.try_lock()) { + std::cout << "\r(Scanning for nodes used in ways: " << (100*blocksProcessed.load()/blocksToProcess.load()) << "%) "; + std::cout.flush(); + ioMutex.unlock(); + } + continue; + } + } + if(phase == ReadPhase::RelationScan) { osmStore.ensureUsedWaysInited(); - bool done = ScanRelations(output, pg, pb); + bool done = ScanRelations(output, pg, pb, wayKeys); if(done) { if (ioMutex.try_lock()) { std::cout << "\r(Scanning for ways used in relations: " << (100*blocksProcessed.load()/blocksToProcess.load()) << "%) "; @@ -347,7 +410,7 @@ bool PbfProcessor::ReadBlock( } if(phase == ReadPhase::Ways) { - bool done = ReadWays(output, pg, pb, locationsOnWays, shard, effectiveShards); + bool done = ReadWays(output, pg, pb, wayKeys, locationsOnWays, shard, effectiveShards); if(done) { output_progress(); ++read_groups; @@ -356,7 +419,7 @@ bool PbfProcessor::ReadBlock( } if(phase == ReadPhase::Relations) { - bool done = ReadRelations(output, pg, pb, blockMetadata, shard, effectiveShards); + bool done = ReadRelations(output, pg, pb, blockMetadata, wayKeys, shard, effectiveShards); if(done) { output_progress(); ++read_groups; @@ -408,7 +471,8 @@ bool blockHasPrimitiveGroupSatisfying( int PbfProcessor::ReadPbfFile( uint shards, bool hasSortTypeThenID, - unordered_set const& nodeKeys, + const SignificantTags& nodeKeys, + const SignificantTags& wayKeys, unsigned int threadNum, const pbfreader_generate_stream& generate_stream, const pbfreader_generate_output& generate_output, @@ -507,7 +571,16 @@ int PbfProcessor::ReadPbfFile( } - std::vector all_phases = { ReadPhase::RelationScan, ReadPhase::Nodes, ReadPhase::Ways, ReadPhase::Relations }; + std::vector all_phases = { ReadPhase::RelationScan }; + if (wayKeys.enabled()) { + osmStore.usedNodes.enable(); + all_phases.push_back(ReadPhase::WayScan); + } + + all_phases.push_back(ReadPhase::Nodes); + all_phases.push_back(ReadPhase::Ways); + all_phases.push_back(ReadPhase::Relations); + for(auto phase: all_phases) { uint effectiveShards = 1; @@ -557,6 +630,7 @@ int PbfProcessor::ReadPbfFile( for (const auto& entry : blocks) { if ((phase == ReadPhase::Nodes && entry.second.hasNodes) || (phase == ReadPhase::RelationScan && entry.second.hasRelations) || + (phase == ReadPhase::WayScan && entry.second.hasWays) || (phase == ReadPhase::Ways && entry.second.hasWays) || (phase == ReadPhase::Relations && entry.second.hasRelations)) filteredBlocks[entry.first] = entry.second; @@ -593,7 +667,7 @@ int PbfProcessor::ReadPbfFile( { for(const std::vector& blockRange: blockRanges) { - boost::asio::post(pool, [=, &blockRange, &blocks, &block_mutex, &nodeKeys]() { + boost::asio::post(pool, [=, &blockRange, &blocks, &block_mutex, &nodeKeys, &wayKeys]() { if (phase == ReadPhase::Nodes) osmStore.nodes.batchStart(); if (phase == ReadPhase::Ways) @@ -603,7 +677,7 @@ int PbfProcessor::ReadPbfFile( auto infile = generate_stream(); auto output = generate_output(); - if(ReadBlock(*infile, *output, indexedBlockMetadata, nodeKeys, locationsOnWays, phase, shard, effectiveShards)) { + if(ReadBlock(*infile, *output, indexedBlockMetadata, nodeKeys, wayKeys, locationsOnWays, phase, shard, effectiveShards)) { const std::lock_guard lock(block_mutex); blocks.erase(indexedBlockMetadata.index); } @@ -622,8 +696,13 @@ int PbfProcessor::ReadPbfFile( #endif } + if(phase == ReadPhase::RelationScan) { + auto output = generate_output(); + output->postScanRelations(); + } if(phase == ReadPhase::Nodes) { osmStore.nodes.finalize(threadNum); + osmStore.usedNodes.clear(); } if(phase == ReadPhase::Ways) { osmStore.ways.finalize(threadNum); diff --git a/src/pooled_string.cpp b/src/pooled_string.cpp index 5a1fe0bc..a7aea1cf 100644 --- a/src/pooled_string.cpp +++ b/src/pooled_string.cpp @@ -9,7 +9,7 @@ namespace PooledStringNS { const uint8_t ShortString = 0b00; const uint8_t HeapString = 0b10; - const uint8_t StdString = 0b11; + const uint8_t DataViewString = 0b11; // Each thread has its own string table, we only take a lock // to push a new table onto the vector. @@ -57,16 +57,16 @@ PooledString::PooledString(const std::string& str) { } } -PooledString::PooledString(const std::string* str) { - storage[0] = StdString << 6; +PooledString::PooledString(const protozero::data_view* str) { + storage[0] = DataViewString << 6; - *(const std::string**)((void*)(storage + 8)) = str; + *(const protozero::data_view**)((void*)(storage + 8)) = str; } bool PooledStringNS::PooledString::operator==(const PooledString& other) const { // NOTE: We have surprising equality semantics! // - // If one of the strings is a StdString, it's value equality. + // If one of the strings is a DataViewString, it's value equality. // // Else, for short strings, you are equal if the strings are equal. // @@ -76,7 +76,7 @@ bool PooledStringNS::PooledString::operator==(const PooledString& other) const { uint8_t kind = storage[0] >> 6; uint8_t otherKind = other.storage[0] >> 6; - if (kind == StdString || otherKind == StdString) { + if (kind == DataViewString || otherKind == DataViewString) { size_t mySize = size(); if (mySize != other.size()) return false; @@ -97,8 +97,8 @@ const char* PooledStringNS::PooledString::data() const { if (kind == ShortString) return (char *)(storage + 1); - if (kind == StdString) { - const std::string* str = *(const std::string**)((void*)(storage + 8)); + if (kind == DataViewString) { + const protozero::data_view* str = *(const protozero::data_view**)((void*)(storage + 8)); return str->data(); } @@ -121,7 +121,7 @@ size_t PooledStringNS::PooledString::size() const { // Otherwise it's stored in the lower 7 bits of the highest byte. return storage[0] & 0b01111111; - const std::string* str = *(const std::string**)((void*)(storage + 8)); + const protozero::data_view* str = *(const protozero::data_view**)((void*)(storage + 8)); return str->size(); } @@ -146,14 +146,14 @@ std::string PooledStringNS::PooledString::toString() const { return rv; } - const std::string* str = *(const std::string**)((void*)(storage + 8)); - return *str; + const protozero::data_view* str = *(const protozero::data_view**)((void*)(storage + 8)); + return std::string(str->data(), str->size()); } void PooledStringNS::PooledString::ensureStringIsOwned() { uint8_t kind = storage[0] >> 6; - if (kind != StdString) + if (kind != DataViewString) return; *this = PooledString(toString()); diff --git a/src/shp_mem_tiles.cpp b/src/shp_mem_tiles.cpp index 564c2040..05383d14 100644 --- a/src/shp_mem_tiles.cpp +++ b/src/shp_mem_tiles.cpp @@ -77,8 +77,8 @@ bool ShpMemTiles::mayIntersect(const std::string& layerName, const Box& box) con uint32_t y1 = lat2tiley(lat1, indexZoom); uint32_t y2 = lat2tiley(lat2, indexZoom); - for (int x = std::min(x1, x2); x < std::max(x1, x2); x++) { - for (int y = std::min(y1, y2); y < std::max(y1, y2); y++) { + for (int x = std::min(x1, x2); x <= std::min((1u << indexZoom) - 1u, std::max(x1, x2)); x++) { + for (int y = std::min(y1, y2); y <= std::min((1u << indexZoom) - 1u, std::max(y1, y2)); y++) { if (bitvec[x * (1 << indexZoom) + y]) return true; } @@ -171,8 +171,8 @@ void ShpMemTiles::StoreGeometry( uint32_t y1 = lat2tiley(lat1, indexZoom); uint32_t y2 = lat2tiley(lat2, indexZoom); - for (int x = std::min(x1, x2); x < std::max(x1, x2); x++) { - for (int y = std::min(y1, y2); y < std::max(y1, y2); y++) { + for (int x = std::min(x1, x2); x <= std::min((1u << indexZoom) - 1u, std::max(x1, x2)); x++) { + for (int y = std::min(y1, y2); y <= std::min((1u << indexZoom) - 1u, std::max(y1, y2)); y++) { bitvec[x * (1 << indexZoom) + y] = true; } } diff --git a/src/significant_tags.cpp b/src/significant_tags.cpp new file mode 100644 index 00000000..7a3663f9 --- /dev/null +++ b/src/significant_tags.cpp @@ -0,0 +1,88 @@ +#include +#include "significant_tags.h" +#include "tag_map.h" + +TagFilter SignificantTags::parseFilter(std::string rawTag) { + TagFilter rv { true }; + + std::string input = rawTag; + + if (input.size() > 0 && input[0] == '~') { + rv.accept = false; + input = input.substr(1); + } + + size_t n = input.find("="); + + if (n == std::string::npos) { + rv.key = input; + return rv; + } + + rv.key = input.substr(0, n); + rv.value = input.substr(n + 1); + + return rv; +} + +SignificantTags::SignificantTags(): enabled_(false) {} + +SignificantTags::SignificantTags(std::vector rawTags): enabled_(true) { + for (const std::string& rawTag : rawTags) { + filters.push_back(parseFilter(rawTag)); + } + + if (filters.empty()) + return; + + bool accept = filters[0].accept; + + size_t i = 0; + for (const auto& filter : filters) { + if (filter.accept != accept) { + throw std::runtime_error("cannot mix reject and accept filters: " + rawTags[0] + ", " + rawTags[i]); + } + i++; + } +} + +bool SignificantTags::enabled() const { return enabled_; } + +bool SignificantTags::filter(const TagMap& tags) const { + if (!enabled_) + return true; + + if (filters.empty()) + return false; + + bool defaultReject = filters[0].accept; + + if (defaultReject) { + // There must be at least one tag matched by the filters. + for (const Tag& tag : tags) { + for (const TagFilter& filter : filters) { + if (filter.key == tag.key && (filter.value.empty() || filter.value == tag.value)) + return true; + } + } + + return false; + } + + // There must be at least one tag not matched by any filters. + for (const Tag& tag : tags) { + // If no filters match this tag, + bool hadMatch = false; + for (const TagFilter& filter : filters) { + if (filter.key == tag.key && (filter.value.empty() || filter.value == tag.value)) { + hadMatch = true; + break; + } + } + + if (!hadMatch) + return true; + } + + return false; +} diff --git a/src/tag_map.cpp b/src/tag_map.cpp new file mode 100644 index 00000000..c3424fb7 --- /dev/null +++ b/src/tag_map.cpp @@ -0,0 +1,173 @@ +#include "tag_map.h" +#include +#include + +TagMap::TagMap() { + keys.resize(16); + key2value.resize(16); + values.resize(16); +} + +void TagMap::reset() { + for (int i = 0; i < 16; i++) { + keys[i].clear(); + key2value[i].clear(); + values[i].clear(); + } +} + +bool TagMap::empty() const { + for (int i = 0; i < keys.size(); i++) + if (keys[i].size() > 0) + return false; + + return true; +} +const std::size_t hashString(const std::string& str) { + // This is a pretty crappy hash function in terms of bit + // avalanching and distribution of output values. + // + // But it's very good in terms of speed, which turns out + // to be the important measure. + std::size_t hash = str.size(); + if (hash >= 4) + hash ^= *(uint32_t*)str.data(); + + return hash; +} + +const std::size_t hashString(const char* str, size_t size) { + // This is a pretty crappy hash function in terms of bit + // avalanching and distribution of output values. + // + // But it's very good in terms of speed, which turns out + // to be the important measure. + std::size_t hash = size; + if (hash >= 4) + hash ^= *(uint32_t*)str; + + return hash; +} + +uint32_t TagMap::ensureString( + std::vector>& vector, + const protozero::data_view& value +) { + std::size_t hash = hashString(value.data(), value.size()); + + const uint16_t shard = hash % vector.size(); + for (int i = 0; i < vector[shard].size(); i++) + if (*(vector[shard][i]) == value) + return shard << 16 | i; + + vector[shard].push_back(&value); + return shard << 16 | (vector[shard].size() - 1); +} + + +void TagMap::addTag(const protozero::data_view& key, const protozero::data_view& value) { + uint32_t valueLoc = ensureString(values, value); + uint32_t keyLoc = ensureString(keys, key); + + + const uint16_t shard = keyLoc >> 16; + const uint16_t pos = keyLoc; + if (key2value[shard].size() <= pos) { + key2value[shard].resize(pos + 1); + } + + key2value[shard][pos] = valueLoc; +} + +int64_t TagMap::getKey(const char* key, size_t size) const { + // Return -1 if key not found, else return its keyLoc. + std::size_t hash = hashString(key, size); + + const uint16_t shard = hash % keys.size(); + for (int i = 0; i < keys[shard].size(); i++) { + const protozero::data_view& candidate = *keys[shard][i]; + if (candidate.size() != size) + continue; + + if (memcmp(candidate.data(), key, size) == 0) + return shard << 16 | i; + } + + return -1; +} + +int64_t TagMap::getValue(const char* value, size_t size) const { + // Return -1 if value not found, else return its valueLoc. + std::size_t hash = hashString(value, size); + + const uint16_t shard = hash % values.size(); + for (int i = 0; i < values[shard].size(); i++) { + const protozero::data_view& candidate = *values[shard][i]; + if (candidate.size() != size) + continue; + + if (memcmp(candidate.data(), value, size) == 0) + return shard << 16 | i; + } + + return -1; +} + +const protozero::data_view* TagMap::getValueFromKey(uint32_t keyLoc) const { + const uint32_t valueLoc = key2value[keyLoc >> 16][keyLoc & 0xFFFF]; + return values[valueLoc >> 16][valueLoc & 0xFFFF]; +} + +const protozero::data_view* TagMap::getValue(uint32_t valueLoc) const { + return values[valueLoc >> 16][valueLoc & 0xFFFF]; +} + +boost::container::flat_map TagMap::exportToBoostMap() const { + boost::container::flat_map rv; + + for (int i = 0; i < keys.size(); i++) { + for (int j = 0; j < keys[i].size(); j++) { + uint32_t valueLoc = key2value[i][j]; + auto key = *keys[i][j]; + auto value = *values[valueLoc >> 16][valueLoc & 0xFFFF]; + rv[std::string(key.data(), key.size())] = std::string(value.data(), value.size()); + } + } + + return rv; +} + +TagMap::Iterator TagMap::begin() const { + size_t shard = 0; + while(keys.size() > shard && keys[shard].size() == 0) + shard++; + + return Iterator{*this, shard, 0}; +} + +TagMap::Iterator TagMap::end() const { + return Iterator{*this, keys.size(), 0}; +} + +bool TagMap::Iterator::operator!=(const Iterator& other) const { + return other.shard != shard || other.offset != offset; +} + +void TagMap::Iterator::operator++() { + ++offset; + if (offset >= map.keys[shard].size()) { + offset = 0; + shard++; + // Advance to the next non-empty shard. + while(map.keys.size() > shard && map.keys[shard].size() == 0) + shard++; + } +} + +Tag TagMap::Iterator::operator*() const { + const uint32_t valueLoc = map.key2value[shard][offset]; + return Tag{ + *map.keys[shard][offset], + *map.getValue(valueLoc) + }; +} diff --git a/src/tile_data.cpp b/src/tile_data.cpp index f78bbdda..407f534a 100644 --- a/src/tile_data.cpp +++ b/src/tile_data.cpp @@ -73,10 +73,21 @@ TileDataSource::TileDataSource(size_t threadNum, unsigned int baseZoom, bool inc } } +thread_local std::vector>* tlsPendingSmallIndexObjects = nullptr; + void TileDataSource::finalize(size_t threadNum) { + uint64_t finalized = 0; + for (const auto& vec : pendingSmallIndexObjects) { + for (const auto& tuple : vec) { + finalized++; + addObjectToSmallIndexUnsafe(std::get<0>(tuple), std::get<1>(tuple), std::get<2>(tuple)); + } + } + + std::cout << "indexed " << finalized << " contended objects" << std::endl; + finalizeObjects(name(), threadNum, baseZoom, objects.begin(), objects.end(), lowZoomObjects); finalizeObjects(name(), threadNum, baseZoom, objectsWithIds.begin(), objectsWithIds.end(), lowZoomObjectsWithIds); - } void TileDataSource::addObjectToSmallIndex(const TileCoordinates& index, const OutputObject& oo, uint64_t id) { @@ -90,8 +101,28 @@ void TileDataSource::addObjectToSmallIndex(const TileCoordinates& index, const O } const size_t z6index = z6x * CLUSTER_ZOOM_WIDTH + z6y; + auto& mutex = objectsMutex[z6index % objectsMutex.size()]; + + if (mutex.try_lock()) { + addObjectToSmallIndexUnsafe(index, oo, id); + mutex.unlock(); + } else { + // add to tlsPendingSmallIndexObjects + if (tlsPendingSmallIndexObjects == nullptr) { + std::lock_guard lock(objectsMutex[0]); + pendingSmallIndexObjects.push_back(std::vector>()); + tlsPendingSmallIndexObjects = &pendingSmallIndexObjects.back(); + } - std::lock_guard lock(objectsMutex[z6index % objectsMutex.size()]); + tlsPendingSmallIndexObjects->push_back(std::make_tuple(index, oo, id)); + } +} + +void TileDataSource::addObjectToSmallIndexUnsafe(const TileCoordinates& index, const OutputObject& oo, uint64_t id) { + // Pick the z6 index + const size_t z6x = index.x / z6OffsetDivisor; + const size_t z6y = index.y / z6OffsetDivisor; + const size_t z6index = z6x * CLUSTER_ZOOM_WIDTH + z6y; if (id == 0 || !includeID) objects[z6index].push_back({ diff --git a/src/tile_worker.cpp b/src/tile_worker.cpp index ec293c9a..89e0ad9b 100644 --- a/src/tile_worker.cpp +++ b/src/tile_worker.cpp @@ -280,7 +280,6 @@ void writeMultiPolygon( if (geom::is_empty(current)) return; -#if BOOST_VERSION >= 105800 geom::validity_failure_type failure; if (verbose && !geom::is_valid(current, failure)) { cout << "output multipolygon has " << boost_validity_error(failure) << endl; @@ -290,11 +289,6 @@ void writeMultiPolygon( else cout << "input multipolygon valid" << endl; } -#else - if (verbose && !geom::is_valid(current)) { - cout << "Output multipolygon is invalid " << endl; - } -#endif vtzero::polygon_feature_builder fbuilder{vtLayer}; diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp index 8c751a79..b18b726f 100644 --- a/src/tilemaker.cpp +++ b/src/tilemaker.cpp @@ -42,6 +42,7 @@ #include "helpers.h" #include "coordinates.h" #include "coordinates_geom.h" +#include "significant_tags.h" #include "attribute_store.h" #include "output_object.h" @@ -117,12 +118,10 @@ int main(const int argc, const char* argv[]) { } - // ---- Read bounding box from first .pbf (if there is one) or mapsplit file + // ---- Read bounding box from first .pbf (if there is one) bool hasClippingBox = false; Box clippingBox; - bool mapsplit = false; - MBTiles mapsplitFile; double minLon=0.0, maxLon=0.0, minLat=0.0, maxLat=0.0; if (!bboxElements.empty()) { hasClippingBox = true; @@ -131,12 +130,6 @@ int main(const int argc, const char* argv[]) { maxLon = bboxElementFromStr(bboxElements.at(2)); maxLat = bboxElementFromStr(bboxElements.at(3)); - } else if (options.inputFiles.size()==1 && (ends_with(options.inputFiles[0], ".mbtiles") || ends_with(options.inputFiles[0], ".sqlite") || ends_with(options.inputFiles[0], ".msf"))) { - mapsplit = true; - mapsplitFile.openForReading(options.inputFiles[0]); - mapsplitFile.readBoundingBox(minLon, maxLon, minLat, maxLat); - hasClippingBox = true; - } else if (options.inputFiles.size()>0) { int ret = ReadPbfBoundingBox(options.inputFiles[0], minLon, maxLon, minLat, maxLat, hasClippingBox); if(ret != 0) return ret; @@ -263,46 +256,46 @@ int main(const int argc, const char* argv[]) { } shpMemTiles.reportSize(); - // ---- Read significant node tags - - vector nodeKeyVec = osmLuaProcessing.GetSignificantNodeKeys(); - unordered_set nodeKeys(nodeKeyVec.begin(), nodeKeyVec.end()); + // ---- Read significant node/way tags + const SignificantTags significantNodeTags = osmLuaProcessing.GetSignificantNodeKeys(); + const SignificantTags significantWayTags = osmLuaProcessing.GetSignificantWayKeys(); // ---- Read all PBFs PbfProcessor pbfProcessor(osmStore); std::vector sortOrders = layers.getSortOrders(); - if (!mapsplit) { - for (auto inputFile : options.inputFiles) { - cout << "Reading .pbf " << inputFile << endl; - ifstream infile(inputFile, ios::in | ios::binary); - if (!infile) { cerr << "Couldn't open .pbf file " << inputFile << endl; return -1; } - - const bool hasSortTypeThenID = PbfHasOptionalFeature(inputFile, OptionSortTypeThenID); - int ret = pbfProcessor.ReadPbfFile( - nodeStore->shards(), - hasSortTypeThenID, - nodeKeys, - options.threadNum, - [&]() { - thread_local std::shared_ptr pbfStream(new ifstream(inputFile, ios::in | ios::binary)); - return pbfStream; - }, - [&]() { - thread_local std::shared_ptr osmLuaProcessing(new OsmLuaProcessing(osmStore, config, layers, options.luaFile, shpMemTiles, osmMemTiles, attributeStore, options.osm.materializeGeometries)); - return osmLuaProcessing; - }, - *nodeStore, - *wayStore - ); - if (ret != 0) return ret; - } - attributeStore.finalize(); - osmMemTiles.reportSize(); - attributeStore.reportSize(); - } + for (auto inputFile : options.inputFiles) { + cout << "Reading .pbf " << inputFile << endl; + ifstream infile(inputFile, ios::in | ios::binary); + if (!infile) { cerr << "Couldn't open .pbf file " << inputFile << endl; return -1; } + + const bool hasSortTypeThenID = PbfHasOptionalFeature(inputFile, OptionSortTypeThenID); + int ret = pbfProcessor.ReadPbfFile( + nodeStore->shards(), + hasSortTypeThenID, + significantNodeTags, + significantWayTags, + options.threadNum, + [&]() { + thread_local std::shared_ptr pbfStream(new ifstream(inputFile, ios::in | ios::binary)); + return pbfStream; + }, + [&]() { + thread_local std::shared_ptr osmLuaProcessing(new OsmLuaProcessing(osmStore, config, layers, options.luaFile, shpMemTiles, osmMemTiles, attributeStore, options.osm.materializeGeometries)); + return osmLuaProcessing; + }, + *nodeStore, + *wayStore + ); + if (ret != 0) return ret; + } + attributeStore.finalize(); + osmMemTiles.reportSize(); + attributeStore.reportSize(); + // ---- Initialise SharedData + SourceList sources = {&osmMemTiles, &shpMemTiles}; class SharedData sharedData(config, layers); sharedData.outputFile = options.outputFile; @@ -320,287 +313,233 @@ int main(const int argc, const char* argv[]) { // ---- Write out data - // If mapsplit, read list of tiles available - unsigned runs=1; - vector> tileList; - if (mapsplit) { - mapsplitFile.readTileList(tileList); - runs = tileList.size(); - } - - for (unsigned run=0; run config.baseZoom) { - cerr << "Mapsplit tiles (zoom " << srcZ << ") must not be greater than basezoom " << config.baseZoom << endl; - return 0; - } else if (srcZ > config.startZoom) { - cout << "Mapsplit tiles (zoom " << srcZ << ") can't write data at zoom level " << config.startZoom << endl; - } - - cout << "Reading tile " << srcZ << ": " << srcX << "," << srcY << " (" << (run+1) << "/" << runs << ")" << endl; - vector pbf = mapsplitFile.readTile(srcZ,srcX,tmsY); - - int ret = pbfProcessor.ReadPbfFile( - nodeStore->shards(), - false, - nodeKeys, - 1, - [&]() { - return make_unique(pbf.data(), pbf.size(), ios::in | ios::binary); - }, - [&]() { - return std::make_unique(osmStore, config, layers, options.luaFile, shpMemTiles, osmMemTiles, attributeStore, options.osm.materializeGeometries); - }, - *nodeStore, - *wayStore - ); - if (ret != 0) return ret; - - tileList.pop_back(); - } + // Launch the pool with threadNum threads + boost::asio::thread_pool pool(options.threadNum); - // Launch the pool with threadNum threads - boost::asio::thread_pool pool(options.threadNum); + // Mutex is hold when IO is performed + std::mutex io_mutex; - // Mutex is hold when IO is performed - std::mutex io_mutex; + // Loop through tiles + std::atomic tilesWritten(0); - // Loop through tiles - std::atomic tilesWritten(0); + for (auto source : sources) { + source->finalize(options.threadNum); + } + // tiles by zoom level - for (auto source : sources) { - source->finalize(options.threadNum); - } - // tiles by zoom level - - // The clipping bbox check is expensive - as an optimization, compute the set of - // z6 tiles that are wholly covered by the clipping box. Membership in this - // set is quick to test. - TileCoordinatesSet coveredZ6Tiles(6); - if (hasClippingBox) { - for (int x = 0; x < 1 << 6; x++) { - for (int y = 0; y < 1 << 6; y++) { - if (boost::geometry::within( - TileBbox(TileCoordinates(x, y), 6, false, false).getTileBox(), - clippingBox - )) - coveredZ6Tiles.set(x, y); - } + // The clipping bbox check is expensive - as an optimization, compute the set of + // z6 tiles that are wholly covered by the clipping box. Membership in this + // set is quick to test. + TileCoordinatesSet coveredZ6Tiles(6); + if (hasClippingBox) { + for (int x = 0; x < 1 << 6; x++) { + for (int y = 0; y < 1 << 6; y++) { + if (boost::geometry::within( + TileBbox(TileCoordinates(x, y), 6, false, false).getTileBox(), + clippingBox + )) + coveredZ6Tiles.set(x, y); } } + } - // For large areas (arbitrarily defined as 100 z6 tiles), use a dense index for pmtiles - if (coveredZ6Tiles.size()>100 && options.outputMode == OptionsParser::OutputMode::PMTiles) { - std::cout << "Using dense index for .pmtiles" << std::endl; - sharedData.pmtiles.isSparse = false; - } + // For large areas (arbitrarily defined as 100 z6 tiles), use a dense index for pmtiles + if (coveredZ6Tiles.size()>100 && options.outputMode == OptionsParser::OutputMode::PMTiles) { + std::cout << "Using dense index for .pmtiles" << std::endl; + sharedData.pmtiles.isSparse = false; + } - std::deque> tileCoordinates; - std::vector zoomResults; - for (uint zoom = 0; zoom <= sharedData.config.endZoom; zoom++) { - zoomResults.push_back(TileCoordinatesSet(zoom)); - } + std::deque> tileCoordinates; + std::vector zoomResults; + for (uint zoom = 0; zoom <= sharedData.config.endZoom; zoom++) { + zoomResults.push_back(TileCoordinatesSet(zoom)); + } - { + { #ifdef CLOCK_MONOTONIC - timespec start, end; - clock_gettime(CLOCK_MONOTONIC, &start); + timespec start, end; + clock_gettime(CLOCK_MONOTONIC, &start); #endif - std::cout << "collecting tiles" << std::flush; - populateTilesAtZoom(sources, zoomResults); + std::cout << "collecting tiles" << std::flush; + populateTilesAtZoom(sources, zoomResults); #ifdef CLOCK_MONOTONIC - clock_gettime(CLOCK_MONOTONIC, &end); - uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; - std::cout << ": " << (uint32_t)(tileNs / 1e6) << "ms"; + clock_gettime(CLOCK_MONOTONIC, &end); + uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; + std::cout << ": " << (uint32_t)(tileNs / 1e6) << "ms"; #endif - } + } - std::cout << ", filtering tiles:" << std::flush; - for (uint zoom=sharedData.config.startZoom; zoom <= sharedData.config.endZoom; zoom++) { - std::cout << " z" << std::to_string(zoom) << std::flush; + std::cout << ", filtering tiles:" << std::flush; + for (uint zoom=sharedData.config.startZoom; zoom <= sharedData.config.endZoom; zoom++) { + std::cout << " z" << std::to_string(zoom) << std::flush; #ifdef CLOCK_MONOTONIC - timespec start, end; - clock_gettime(CLOCK_MONOTONIC, &start); + timespec start, end; + clock_gettime(CLOCK_MONOTONIC, &start); #endif - const auto& zoomResult = zoomResults[zoom]; - int numTiles = 0; - for (int x = 0; x < 1 << zoom; x++) { - for (int y = 0; y < 1 << zoom; y++) { - if (!zoomResult.test(x, y)) - continue; - - // If we're constrained to a source tile, check we're within it - if (srcZ > -1) { - int xAtSrcZ = x / pow(2, zoom-srcZ); - int yAtSrcZ = y / pow(2, zoom-srcZ); - if (xAtSrcZ != srcX || yAtSrcZ != srcY) continue; - } - - if (hasClippingBox) { - bool isInAWhollyCoveredZ6Tile = false; - if (zoom >= 6) { - TileCoordinate z6x = x / (1 << (zoom - 6)); - TileCoordinate z6y = y / (1 << (zoom - 6)); - isInAWhollyCoveredZ6Tile = coveredZ6Tiles.test(z6x, z6y); - } - - if(!isInAWhollyCoveredZ6Tile && !boost::geometry::intersects(TileBbox(TileCoordinates(x, y), zoom, false, false).getTileBox(), clippingBox)) - continue; + const auto& zoomResult = zoomResults[zoom]; + int numTiles = 0; + for (int x = 0; x < 1 << zoom; x++) { + for (int y = 0; y < 1 << zoom; y++) { + if (!zoomResult.test(x, y)) + continue; + + if (hasClippingBox) { + bool isInAWhollyCoveredZ6Tile = false; + if (zoom >= 6) { + TileCoordinate z6x = x / (1 << (zoom - 6)); + TileCoordinate z6y = y / (1 << (zoom - 6)); + isInAWhollyCoveredZ6Tile = coveredZ6Tiles.test(z6x, z6y); } - tileCoordinates.push_back(std::make_pair(zoom, TileCoordinates(x, y))); - numTiles++; + if(!isInAWhollyCoveredZ6Tile && !boost::geometry::intersects(TileBbox(TileCoordinates(x, y), zoom, false, false).getTileBox(), clippingBox)) + continue; } + + tileCoordinates.push_back(std::make_pair(zoom, TileCoordinates(x, y))); + numTiles++; } + } - std::cout << " (" << numTiles; + std::cout << " (" << numTiles; #ifdef CLOCK_MONOTONIC - clock_gettime(CLOCK_MONOTONIC, &end); - uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; - std::cout << ", " << (uint32_t)(tileNs / 1e6) << "ms"; + clock_gettime(CLOCK_MONOTONIC, &end); + uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; + std::cout << ", " << (uint32_t)(tileNs / 1e6) << "ms"; #endif - std::cout << ")" << std::flush; - } - zoomResults.clear(); - - std::cout << std::endl; - - // Cluster tiles: breadth-first for z0..z5, depth-first for z6 - const size_t baseZoom = config.baseZoom; - boost::sort::block_indirect_sort( - tileCoordinates.begin(), tileCoordinates.end(), - [baseZoom](auto const &a, auto const &b) { - const auto aZoom = a.first; - const auto bZoom = b.first; - const auto aX = a.second.x; - const auto aY = a.second.y; - const auto bX = b.second.x; - const auto bY = b.second.y; - const bool aLowZoom = aZoom < CLUSTER_ZOOM; - const bool bLowZoom = bZoom < CLUSTER_ZOOM; - - // Breadth-first for z0..5 - if (aLowZoom != bLowZoom) - return aLowZoom; - - if (aLowZoom && bLowZoom) { - if (aZoom != bZoom) - return aZoom < bZoom; - - if (aX != bX) - return aX < bX; - - return aY < bY; - } - - for (size_t z = CLUSTER_ZOOM; z <= baseZoom; z++) { - // Translate both a and b to zoom z, compare. - // First, sanity check: can we translate it to this zoom? - if (aZoom < z || bZoom < z) { - return aZoom < bZoom; - } + std::cout << ")" << std::flush; + } + zoomResults.clear(); + + std::cout << std::endl; + + // Cluster tiles: breadth-first for z0..z5, depth-first for z6 + const size_t baseZoom = config.baseZoom; + boost::sort::block_indirect_sort( + tileCoordinates.begin(), tileCoordinates.end(), + [baseZoom](auto const &a, auto const &b) { + const auto aZoom = a.first; + const auto bZoom = b.first; + const auto aX = a.second.x; + const auto aY = a.second.y; + const auto bX = b.second.x; + const auto bY = b.second.y; + const bool aLowZoom = aZoom < CLUSTER_ZOOM; + const bool bLowZoom = bZoom < CLUSTER_ZOOM; + + // Breadth-first for z0..5 + if (aLowZoom != bLowZoom) + return aLowZoom; + + if (aLowZoom && bLowZoom) { + if (aZoom != bZoom) + return aZoom < bZoom; + + if (aX != bX) + return aX < bX; + + return aY < bY; + } - const auto aXz = aX / (1 << (aZoom - z)); - const auto aYz = aY / (1 << (aZoom - z)); - const auto bXz = bX / (1 << (bZoom - z)); - const auto bYz = bY / (1 << (bZoom - z)); + for (size_t z = CLUSTER_ZOOM; z <= baseZoom; z++) { + // Translate both a and b to zoom z, compare. + // First, sanity check: can we translate it to this zoom? + if (aZoom < z || bZoom < z) { + return aZoom < bZoom; + } - if (aXz != bXz) - return aXz < bXz; + const auto aXz = aX / (1 << (aZoom - z)); + const auto aYz = aY / (1 << (aZoom - z)); + const auto bXz = bX / (1 << (bZoom - z)); + const auto bYz = bY / (1 << (bZoom - z)); - if (aYz != bYz) - return aYz < bYz; - } + if (aXz != bXz) + return aXz < bXz; - return false; - }, - options.threadNum); - - std::size_t batchSize = 0; - for(std::size_t startIndex = 0; startIndex < tileCoordinates.size(); startIndex += batchSize) { - // Compute how many tiles should be assigned to this batch -- - // higher-zoom tiles are cheaper to compute, lower-zoom tiles more expensive. - batchSize = 0; - size_t weight = 0; - while (weight < 1000 && startIndex + batchSize < tileCoordinates.size()) { - const auto& zoom = tileCoordinates[startIndex + batchSize].first; - if (zoom > 12) - weight++; - else if (zoom > 11) - weight += 10; - else if (zoom > 10) - weight += 100; - else - weight += 1000; - - batchSize++; + if (aYz != bYz) + return aYz < bYz; } - boost::asio::post(pool, [=, &tileCoordinates, &pool, &sharedData, &sources, &attributeStore, &io_mutex, &tilesWritten]() { - std::vector tileTimings; - std::size_t endIndex = std::min(tileCoordinates.size(), startIndex + batchSize); - for(std::size_t i = startIndex; i < endIndex; ++i) { - unsigned int zoom = tileCoordinates[i].first; - TileCoordinates coords = tileCoordinates[i].second; - -#ifdef CLOCK_MONOTONIC - timespec start, end; - if (options.logTileTimings) - clock_gettime(CLOCK_MONOTONIC, &start); -#endif + return false; + }, + options.threadNum); + + std::size_t batchSize = 0; + for(std::size_t startIndex = 0; startIndex < tileCoordinates.size(); startIndex += batchSize) { + // Compute how many tiles should be assigned to this batch -- + // higher-zoom tiles are cheaper to compute, lower-zoom tiles more expensive. + batchSize = 0; + size_t weight = 0; + while (weight < 1000 && startIndex + batchSize < tileCoordinates.size()) { + const auto& zoom = tileCoordinates[startIndex + batchSize].first; + if (zoom > 12) + weight++; + else if (zoom > 11) + weight += 10; + else if (zoom > 10) + weight += 100; + else + weight += 1000; + + batchSize++; + } - std::vector> data; - for (auto source : sources) { - data.emplace_back(source->getObjectsForTile(sortOrders, zoom, coords)); - } - outputProc(sharedData, sources, attributeStore, data, coords, zoom); + boost::asio::post(pool, [=, &tileCoordinates, &pool, &sharedData, &sources, &attributeStore, &io_mutex, &tilesWritten]() { + std::vector tileTimings; + std::size_t endIndex = std::min(tileCoordinates.size(), startIndex + batchSize); + for(std::size_t i = startIndex; i < endIndex; ++i) { + unsigned int zoom = tileCoordinates[i].first; + TileCoordinates coords = tileCoordinates[i].second; #ifdef CLOCK_MONOTONIC - if (options.logTileTimings) { - clock_gettime(CLOCK_MONOTONIC, &end); - uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; - std::string output = "z" + std::to_string(zoom) + "/" + std::to_string(coords.x) + "/" + std::to_string(coords.y) + " took " + std::to_string(tileNs/1e6) + " ms"; - tileTimings.push_back(output); - } + timespec start, end; + if (options.logTileTimings) + clock_gettime(CLOCK_MONOTONIC, &start); #endif + + std::vector> data; + for (auto source : sources) { + data.emplace_back(source->getObjectsForTile(sortOrders, zoom, coords)); } + outputProc(sharedData, sources, attributeStore, data, coords, zoom); +#ifdef CLOCK_MONOTONIC if (options.logTileTimings) { - const std::lock_guard lock(io_mutex); - std::cout << std::endl; - for (const auto& output : tileTimings) - std::cout << output << std::endl; + clock_gettime(CLOCK_MONOTONIC, &end); + uint64_t tileNs = 1e9 * (end.tv_sec - start.tv_sec) + end.tv_nsec - start.tv_nsec; + std::string output = "z" + std::to_string(zoom) + "/" + std::to_string(coords.x) + "/" + std::to_string(coords.y) + " took " + std::to_string(tileNs/1e6) + " ms"; + tileTimings.push_back(output); } +#endif + } - tilesWritten += (endIndex - startIndex); - - if (io_mutex.try_lock()) { - // Show progress grouped by z6 (or lower) - size_t z = tileCoordinates[startIndex].first; - size_t x = tileCoordinates[startIndex].second.x; - size_t y = tileCoordinates[startIndex].second.y; - if (z > CLUSTER_ZOOM) { - x = x / (1 << (z - CLUSTER_ZOOM)); - y = y / (1 << (z - CLUSTER_ZOOM)); - z = CLUSTER_ZOOM; - } - cout << "z" << z << "/" << x << "/" << y << ", writing tile " << tilesWritten.load() << " of " << tileCoordinates.size() << " \r" << std::flush; - io_mutex.unlock(); + if (options.logTileTimings) { + const std::lock_guard lock(io_mutex); + std::cout << std::endl; + for (const auto& output : tileTimings) + std::cout << output << std::endl; + } + + tilesWritten += (endIndex - startIndex); + + if (io_mutex.try_lock()) { + // Show progress grouped by z6 (or lower) + size_t z = tileCoordinates[startIndex].first; + size_t x = tileCoordinates[startIndex].second.x; + size_t y = tileCoordinates[startIndex].second.y; + if (z > CLUSTER_ZOOM) { + x = x / (1 << (z - CLUSTER_ZOOM)); + y = y / (1 << (z - CLUSTER_ZOOM)); + z = CLUSTER_ZOOM; } - }); - } - // Wait for all tasks in the pool to complete. - pool.join(); + cout << "z" << z << "/" << x << "/" << y << ", writing tile " << tilesWritten.load() << " of " << tileCoordinates.size() << " \r" << std::flush; + io_mutex.unlock(); + } + }); } + // Wait for all tasks in the pool to complete. + pool.join(); // ---- Close tileset diff --git a/test/deque_map.test.cpp b/test/deque_map.test.cpp index 23a3d3cc..28023542 100644 --- a/test/deque_map.test.cpp +++ b/test/deque_map.test.cpp @@ -25,9 +25,13 @@ MU_TEST(test_deque_map) { mu_check(strs.size() == 4); mu_check(strs.at(0) == "foo"); + mu_check(strs[0] == "foo"); mu_check(strs.at(1) == "bar"); + mu_check(strs[1] == "bar"); mu_check(strs.at(2) == "aardvark"); + mu_check(strs[2] == "aardvark"); mu_check(strs.at(3) == "quux"); + mu_check(strs[3] == "quux"); std::vector rv; for (std::string x : strs) { diff --git a/test/options_parser.test.cpp b/test/options_parser.test.cpp index e230fc0d..a3d71b4b 100644 --- a/test/options_parser.test.cpp +++ b/test/options_parser.test.cpp @@ -49,20 +49,19 @@ MU_TEST(test_options_parser) { mu_check(opts.inputFiles[0] == "ontario.pbf"); mu_check(opts.outputFile == "foo.mbtiles"); mu_check(opts.outputMode == OutputMode::MBTiles); - mu_check(opts.osm.materializeGeometries); + mu_check(!opts.osm.materializeGeometries); mu_check(!opts.osm.shardStores); } - // --lazy-geometries overrides default + // --fast without store should have materialized geometries { - std::vector args = {"--output", "foo.mbtiles", "--input", "ontario.pbf", "--lazy-geometries"}; + std::vector args = {"--output", "foo.mbtiles", "--input", "ontario.pbf", "--fast"}; auto opts = parse(args); mu_check(opts.inputFiles.size() == 1); mu_check(opts.inputFiles[0] == "ontario.pbf"); mu_check(opts.outputFile == "foo.mbtiles"); mu_check(opts.outputMode == OutputMode::MBTiles); - mu_check(!opts.osm.materializeGeometries); - mu_check(opts.osm.lazyGeometries); + mu_check(opts.osm.materializeGeometries); mu_check(!opts.osm.shardStores); } diff --git a/test/pooled_string.test.cpp b/test/pooled_string.test.cpp index 91fb2da5..93a05843 100644 --- a/test/pooled_string.test.cpp +++ b/test/pooled_string.test.cpp @@ -23,13 +23,15 @@ MU_TEST(test_pooled_string) { mu_check(big.toString() != big2.toString()); std::string shortString("short"); + protozero::data_view shortStringView = { shortString.data(), shortString.size() }; std::string longString("this is a very long string"); + protozero::data_view longStringView = { longString.data(), longString.size() }; - PooledString stdShortString(&shortString); + PooledString stdShortString(&shortStringView); mu_check(stdShortString.size() == 5); mu_check(stdShortString.toString() == "short"); - PooledString stdLongString(&longString); + PooledString stdLongString(&longStringView); mu_check(stdLongString.size() == 26); mu_check(stdLongString.toString() == "this is a very long string"); diff --git a/test/significant_tags.test.cpp b/test/significant_tags.test.cpp new file mode 100644 index 00000000..33e26948 --- /dev/null +++ b/test/significant_tags.test.cpp @@ -0,0 +1,119 @@ +#include +#include "external/minunit.h" +#include "significant_tags.h" +#include "tag_map.h" + +MU_TEST(test_parse_filter) { + { + TagFilter expected{true, "foo", ""}; + mu_check(SignificantTags::parseFilter("foo") == expected); + } + + { + TagFilter expected{false, "foo", ""}; + mu_check(SignificantTags::parseFilter("~foo") == expected); + } + + { + TagFilter expected{true, "foo", "bar"}; + mu_check(SignificantTags::parseFilter("foo=bar") == expected); + } + + { + TagFilter expected{false, "foo", "bar"}; + mu_check(SignificantTags::parseFilter("~foo=bar") == expected); + } + +} + +MU_TEST(test_invalid_significant_tags) { + bool threw = false; + try { + // Filters must be all accept, or all reject, not a mix. + SignificantTags tags({"a", "~b"}); + } catch (...) { + threw = true; + } + + mu_check(threw); +} + +MU_TEST(test_significant_tags) { + const std::string building = "building"; + const std::string yes = "yes"; + const std::string name = "name"; + const std::string nameValue = "Some name"; + const std::string power = "power"; + const std::string tower = "tower"; + + // If created with no list, it's not enabled and all things pass filter. + // This is the case when people omit `node_keys` or `way_keys`. + { + SignificantTags tags; + TagMap map; + mu_check(tags.filter(map)); + } + + // If created with empty list, it rejects all things. + // This is the case when people write `way_keys = {}`, e.g. when creating + // an extract that only parses nodes. + { + std::vector empty; + SignificantTags tags(empty); + TagMap map; + mu_check(!tags.filter(map)); + } + + // If created in default-accept mode, it accepts anything with an unmatched tag. + // This is the case when people write `way_keys = {"-building"}` + { + std::vector defaultAccept{"~building"}; + SignificantTags tags(defaultAccept); + + { + TagMap map; + map.addTag(building, yes); + mu_check(!tags.filter(map)); + } + + { + TagMap map; + map.addTag(building, yes); + map.addTag(name, nameValue); + mu_check(tags.filter(map)); + } + + } + + // If created in default-reject mode, it accepts anything with a matched tag. + // This is the case when people write `way_keys = {"power=tower"}` + { + std::vector defaultReject{"power=tower"}; + SignificantTags tags(defaultReject); + + { + TagMap map; + mu_check(!tags.filter(map)); + } + + { + TagMap map; + map.addTag(power, tower); + mu_check(tags.filter(map)); + } + + } +} + +MU_TEST_SUITE(test_suite_significant_tags) { + MU_RUN_TEST(test_parse_filter); + MU_RUN_TEST(test_significant_tags); + MU_RUN_TEST(test_invalid_significant_tags); +} + +int main() { + MU_RUN_SUITE(test_suite_significant_tags); + MU_REPORT(); + return MU_EXIT_CODE; +} + diff --git a/test/sorted_way_store.test.cpp b/test/sorted_way_store.test.cpp index 65d34816..02749924 100644 --- a/test/sorted_way_store.test.cpp +++ b/test/sorted_way_store.test.cpp @@ -15,7 +15,9 @@ class TestNodeStore : public NodeStore { void insert(const std::vector>& elements) override {} bool contains(size_t shard, NodeID id) const override { return true; } - size_t shard() const override { return 0; } + NodeStore& shard(size_t shard) override { return *this; } + const NodeStore& shard(size_t shard) const override { return *this; } + size_t shards() const override { return 1; } };