From e005a1ad60567ebbf33f6006cf5df4e061a6e3eb Mon Sep 17 00:00:00 2001 From: junderwood Date: Wed, 13 Apr 2022 15:06:10 -0700 Subject: [PATCH 01/56] Hide non NE countries with a non rendering min zoom --- vectordatasource/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index 623d7ce47..fa1988386 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -8912,7 +8912,7 @@ def tags_set_ne_min_max_zoom(ctx): # way, we probably don't want to see them at zoom, which is lower # than most of the curated NE min zooms. see issue #1826 for more # information. - props['min_zoom'] = max(6, props['min_zoom']) + props['min_zoom'] = max(20, props['min_zoom']) elif props.get('kind') == 'region': props['min_zoom'] = max(8, props['min_zoom']) From 210e038ef87b749905a2da7e395d191f035ef1e5 Mon Sep 17 00:00:00 2001 From: junderwood Date: Wed, 13 Apr 2022 18:06:01 -0700 Subject: [PATCH 02/56] different value --- vectordatasource/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index fa1988386..080395ad0 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -8912,7 +8912,7 @@ def tags_set_ne_min_max_zoom(ctx): # way, we probably don't want to see them at zoom, which is lower # than most of the curated NE min zooms. see issue #1826 for more # information. - props['min_zoom'] = max(20, props['min_zoom']) + props['min_zoom'] = max(16, props['min_zoom']) elif props.get('kind') == 'region': props['min_zoom'] = max(8, props['min_zoom']) From 91920b7765061cb973deeecaee5362c859a1b264 Mon Sep 17 00:00:00 2001 From: junderwood Date: Wed, 13 Apr 2022 18:07:59 -0700 Subject: [PATCH 03/56] Revert "different value" This reverts commit 210e038ef87b749905a2da7e395d191f035ef1e5. --- vectordatasource/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index 080395ad0..fa1988386 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -8912,7 +8912,7 @@ def tags_set_ne_min_max_zoom(ctx): # way, we probably don't want to see them at zoom, which is lower # than most of the curated NE min zooms. see issue #1826 for more # information. - props['min_zoom'] = max(16, props['min_zoom']) + props['min_zoom'] = max(20, props['min_zoom']) elif props.get('kind') == 'region': props['min_zoom'] = max(8, props['min_zoom']) From 3dc0880ca9915248c9a0fe9568989eb075c10129 Mon Sep 17 00:00:00 2001 From: junderwood Date: Wed, 13 Apr 2022 18:09:03 -0700 Subject: [PATCH 04/56] different value --- vectordatasource/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index fa1988386..8ee89ca37 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -8912,7 +8912,7 @@ def tags_set_ne_min_max_zoom(ctx): # way, we probably don't want to see them at zoom, which is lower # than most of the curated NE min zooms. see issue #1826 for more # information. - props['min_zoom'] = max(20, props['min_zoom']) + props['min_zoom'] = max(10, props['min_zoom']) elif props.get('kind') == 'region': props['min_zoom'] = max(8, props['min_zoom']) From 9f78406b6d3a3173522f182644c828f818469ac0 Mon Sep 17 00:00:00 2001 From: junderwood Date: Thu, 14 Apr 2022 09:37:15 -0700 Subject: [PATCH 05/56] testing --- vectordatasource/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index 8ee89ca37..2958793f7 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -8912,7 +8912,7 @@ def tags_set_ne_min_max_zoom(ctx): # way, we probably don't want to see them at zoom, which is lower # than most of the curated NE min zooms. see issue #1826 for more # information. - props['min_zoom'] = max(10, props['min_zoom']) + props['min_zoom'] = max(8, props['min_zoom']) elif props.get('kind') == 'region': props['min_zoom'] = max(8, props['min_zoom']) From ee7ccbf9c7ceae11fdb651490db6f5f78b4edae1 Mon Sep 17 00:00:00 2001 From: junderwood Date: Wed, 20 Apr 2022 13:52:09 -0700 Subject: [PATCH 06/56] set value to non rendering --- vectordatasource/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index 2958793f7..fa1988386 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -8912,7 +8912,7 @@ def tags_set_ne_min_max_zoom(ctx): # way, we probably don't want to see them at zoom, which is lower # than most of the curated NE min zooms. see issue #1826 for more # information. - props['min_zoom'] = max(8, props['min_zoom']) + props['min_zoom'] = max(20, props['min_zoom']) elif props.get('kind') == 'region': props['min_zoom'] = max(8, props['min_zoom']) From 3bb7e0d580b66a6c630f7f1cc80b2f6d41634b7c Mon Sep 17 00:00:00 2001 From: junderwood Date: Thu, 21 Apr 2022 09:17:52 -0700 Subject: [PATCH 07/56] Update kind rather than min_zoom --- integration-test/1826-unrecognised-country-labels.py | 2 +- vectordatasource/transform.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/integration-test/1826-unrecognised-country-labels.py b/integration-test/1826-unrecognised-country-labels.py index c011290c0..c338e4f4a 100644 --- a/integration-test/1826-unrecognised-country-labels.py +++ b/integration-test/1826-unrecognised-country-labels.py @@ -37,7 +37,7 @@ def test_country_no_ne(self): self.assert_has_feature( z, x, y, 'places', { - 'kind': 'country', + 'kind': 'unrecognized', 'min_zoom': 6, }) diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index fa1988386..86b83ff09 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -8912,7 +8912,8 @@ def tags_set_ne_min_max_zoom(ctx): # way, we probably don't want to see them at zoom, which is lower # than most of the curated NE min zooms. see issue #1826 for more # information. - props['min_zoom'] = max(20, props['min_zoom']) + props['kind'] = 'unrecognised' + props['min_zoom'] = max(6, props['min_zoom']) elif props.get('kind') == 'region': props['min_zoom'] = max(8, props['min_zoom']) From 0976306e64812bf4533f06b36405d4e5085f27a6 Mon Sep 17 00:00:00 2001 From: junderwood Date: Mon, 2 May 2022 15:01:40 -0700 Subject: [PATCH 08/56] updates --- vectordatasource/transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index 86b83ff09..cebfe2b35 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -8912,8 +8912,8 @@ def tags_set_ne_min_max_zoom(ctx): # way, we probably don't want to see them at zoom, which is lower # than most of the curated NE min zooms. see issue #1826 for more # information. - props['kind'] = 'unrecognised' - props['min_zoom'] = max(6, props['min_zoom']) + props['kind'] = 'unrecognized' + props['min_zoom'] = max(8, props['min_zoom']) elif props.get('kind') == 'region': props['min_zoom'] = max(8, props['min_zoom']) From e853595dd18bd16000bf559b4f0c1e2898dec33f Mon Sep 17 00:00:00 2001 From: Travis Grigsby Date: Thu, 28 Apr 2022 16:03:59 -0700 Subject: [PATCH 09/56] updating to newest date in assets.yaml --- data/assets.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/assets.yaml b/data/assets.yaml index abb0e32bc..b0a4fdb6a 100644 --- a/data/assets.yaml +++ b/data/assets.yaml @@ -1,5 +1,5 @@ bucket: nextzen-tile-assets -datestamp: 20220426 +datestamp: 20220428 shapefiles: From fc1f2c62ac9075df6d3b573c4fbb45823f5229d8 Mon Sep 17 00:00:00 2001 From: Travis Grigsby Date: Fri, 29 Apr 2022 12:49:24 -0700 Subject: [PATCH 10/56] updating assets.yaml --- data/assets.yaml | 64 ++++++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/data/assets.yaml b/data/assets.yaml index b0a4fdb6a..0404bcc01 100644 --- a/data/assets.yaml +++ b/data/assets.yaml @@ -1,5 +1,5 @@ bucket: nextzen-tile-assets -datestamp: 20220428 +datestamp: 20220429 shapefiles: @@ -55,98 +55,98 @@ shapefiles: url: http://s3.amazonaws.com/tilezen-assets/curated/admin_areas_20180409.zip - name: ne_110m_lakes - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/physical/ne_110m_lakes.zip + url: https://naciscdn.org/naturalearth/5.1.0/110m/physical/ne_110m_lakes.zip - name: ne_50m_lakes - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/physical/ne_50m_lakes.zip + url: https://naciscdn.org/naturalearth/5.1.0/50m/physical/ne_50m_lakes.zip - name: ne_10m_lakes - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_lakes.zip + url: https://naciscdn.org/naturalearth/5.1.0/10m/physical/ne_10m_lakes.zip - name: ne_110m_ocean - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/physical/ne_110m_ocean.zip + url: https://naciscdn.org/naturalearth/5.1.0/110m/physical/ne_110m_ocean.zip - name: ne_50m_ocean - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/physical/ne_50m_ocean.zip + url: https://naciscdn.org/naturalearth/5.1.0/50m/physical/ne_50m_ocean.zip - name: ne_10m_ocean - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_ocean.zip + url: https://naciscdn.org/naturalearth/5.1.0/10m/physical/ne_10m_ocean.zip - name: ne_50m_playas - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/physical/ne_50m_playas.zip + url: https://naciscdn.org/naturalearth/5.1.0/50m/physical/ne_50m_playas.zip - name: ne_10m_playas - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_playas.zip + url: https://naciscdn.org/naturalearth/5.1.0/10m/physical/ne_10m_playas.zip - name: ne_50m_urban_areas - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_urban_areas.zip + url: https://naciscdn.org/naturalearth/5.1.0/50m/cultural/ne_50m_urban_areas.zip - name: ne_10m_urban_areas - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_urban_areas.zip + url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_urban_areas.zip - name: ne_110m_land - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/physical/ne_110m_land.zip + url: https://naciscdn.org/naturalearth/5.1.0/110m/physical/ne_110m_land.zip - name: ne_50m_land - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/physical/ne_50m_land.zip + url: https://naciscdn.org/naturalearth/5.1.0/50m/physical/ne_50m_land.zip - name: ne_10m_land - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_land.zip + url: https://naciscdn.org/naturalearth/5.1.0/10m/physical/ne_10m_land.zip tile: true - name: ne_10m_populated_places - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_populated_places.zip + url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_populated_places.zip - name: ne_110m_admin_0_boundary_lines_land - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/cultural/ne_110m_admin_0_boundary_lines_land.zip + url: https://naciscdn.org/naturalearth/5.1.0/110m/cultural/ne_110m_admin_0_boundary_lines_land.zip - name: ne_50m_admin_0_boundary_lines_land - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_0_boundary_lines_land.zip + url: https://naciscdn.org/naturalearth/5.1.0/50m/cultural/ne_50m_admin_0_boundary_lines_land.zip - name: ne_10m_admin_0_boundary_lines_land - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_boundary_lines_land.zip + url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_boundary_lines_land.zip - name: ne_10m_admin_0_boundary_lines_map_units - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_boundary_lines_map_units.zip + url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_boundary_lines_map_units.zip - name: ne_50m_admin_1_states_provinces_lines - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_1_states_provinces_lines.zip + url: https://naciscdn.org/naturalearth/5.1.0/50m/cultural/ne_50m_admin_1_states_provinces_lines.zip - name: ne_10m_admin_1_states_provinces_lines - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_1_states_provinces_lines.zip + url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_1_states_provinces_lines.zip - name: ne_50m_admin_0_boundary_lines_disputed_areas - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_0_boundary_lines_disputed_areas.zip + url: https://naciscdn.org/naturalearth/5.1.0/50m/cultural/ne_50m_admin_0_boundary_lines_disputed_areas.zip - name: ne_50m_admin_0_boundary_lines_maritime_indicator_chn - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_0_boundary_lines_maritime_indicator_chn.zip + url: https://naciscdn.org/naturalearth/5.1.0/50m/cultural/ne_50m_admin_0_boundary_lines_maritime_indicator_chn.zip - name: ne_10m_admin_0_boundary_lines_disputed_areas - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_boundary_lines_disputed_areas.zip + url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_boundary_lines_disputed_areas.zip - name: ne_10m_admin_0_boundary_lines_maritime_indicator_chn - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_boundary_lines_maritime_indicator_chn.zip + url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_boundary_lines_maritime_indicator_chn.zip - name: ne_10m_roads - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_roads.zip + url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_roads.zip - name: ne_110m_coastline - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/physical/ne_110m_coastline.zip + url: https://naciscdn.org/naturalearth/5.1.0/110m/physical/ne_110m_coastline.zip - name: ne_50m_coastline - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/physical/ne_50m_coastline.zip + url: https://naciscdn.org/naturalearth/5.1.0/50m/physical/ne_50m_coastline.zip - name: ne_10m_coastline - url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_coastline.zip + url: https://naciscdn.org/naturalearth/5.1.0/10m/physical/ne_10m_coastline.zip - name: ne_10m_admin_0_countries - url: https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_countries.zip + url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_countries.zip - name: ne_10m_admin_0_map_units - url: https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_map_units.zip + url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_map_units.zip - name: ne_10m_admin_1_states_provinces - url: https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_1_states_provinces.zip + url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_1_states_provinces.zip wikidata-queries: - name: aerodrome_passenger_count From 776ff679e3acbf91b45e93d238260deff94381a2 Mon Sep 17 00:00:00 2001 From: Travis Grigsby Date: Wed, 4 May 2022 15:59:41 -0700 Subject: [PATCH 11/56] updating assets.yaml to pull from 5.1.0 now --- data/assets.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/data/assets.yaml b/data/assets.yaml index 0404bcc01..683929015 100644 --- a/data/assets.yaml +++ b/data/assets.yaml @@ -142,6 +142,12 @@ shapefiles: - name: ne_10m_admin_0_countries url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_countries.zip + - name: ne_10m_admin_0_countries_iso + url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_countries_iso.zip + + - name: ne_10m_admin_0_countries_tlc + url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_countries_tlc.zip + - name: ne_10m_admin_0_map_units url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_map_units.zip From b05b25d1a0c4a431436e065e8c4e5192fafad5a1 Mon Sep 17 00:00:00 2001 From: Travis Grigsby Date: Wed, 4 May 2022 16:06:44 -0700 Subject: [PATCH 12/56] split lookup depending on whether it's a country (#2075) --- data/functions.sql | 36 +++++++++++++++++---------------- queries/planet_osm_point.jinja2 | 2 +- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/data/functions.sql b/data/functions.sql index 626d0c4fd..4af7d07fd 100644 --- a/data/functions.sql +++ b/data/functions.sql @@ -996,7 +996,7 @@ $$ LANGUAGE plpgsql IMMUTABLE; -- returns a JSONB object containing __ne_min_zoom and __ne_max_zoom set to the -- label min and max zoom of any matching row from the Natural Earth countries, -- map units and states/provinces themes. -CREATE OR REPLACE FUNCTION tz_get_ne_min_max_zoom(wikidata_id TEXT) +CREATE OR REPLACE FUNCTION tz_get_ne_min_max_zoom(wikidata_id TEXT, place_tag TEXT) RETURNS JSONB AS $$ DECLARE min_zoom REAL; @@ -1006,28 +1006,30 @@ BEGIN RETURN '{}'::jsonb; END IF; - -- first, try the countries table - SELECT - min_label, max_label INTO min_zoom, max_zoom - FROM ne_10m_admin_0_countries c - WHERE c.wikidataid = wikidata_id; + -- if it's a country, only look it up in the iso and tlc tables + IF place_tag='country' OR place_tag='unrecognized' THEN + SELECT + min_label, max_label INTO min_zoom, max_zoom + FROM ne_10m_admin_0_countries_iso i + WHERE i.wikidataid = wikidata_id; - -- if that fails, try map_units (which contains some sub-country but super- - -- state level stuff such as England, Scotland and Wales). - IF NOT FOUND THEN - SELECT - min_label, max_label INTO min_zoom, max_zoom - FROM ne_10m_admin_0_map_units mu - WHERE mu.wikidataid = wikidata_id; + IF NOT FOUND THEN + SELECT + min_label, max_label INTO min_zoom, max_zoom + FROM ne_10m_admin_0_countries_tlc t + WHERE t.wikidataid = wikidata_id; + END IF; + + IF NOT FOUND THEN + RETURN '{}'::jsonb; + END IF; END IF; - -- try states and provinces - IF NOT FOUND THEN - SELECT + -- try states and provinces if it's not a country + SELECT min_label, max_label INTO min_zoom, max_zoom FROM ne_10m_admin_1_states_provinces sp WHERE sp.wikidataid = wikidata_id; - END IF; -- finally, try localities -- There is no concept of max_zoom for ne_10m_populated_places diff --git a/queries/planet_osm_point.jinja2 b/queries/planet_osm_point.jinja2 index 5cb48f0c4..2e9e332ec 100644 --- a/queries/planet_osm_point.jinja2 +++ b/queries/planet_osm_point.jinja2 @@ -27,7 +27,7 @@ SELECT THEN jsonb_build_object( 'min_zoom', mz_places_min_zoom ) || - tz_get_ne_min_max_zoom(tags->'wikidata') + tz_get_ne_min_max_zoom(tags->'wikidata', tags->'place') || tz_get_ne_pop_min_max(tags->'wikidata') END AS __places_properties__, From b810954396fe2b6a1cb2ae3802eddaf57a1e9490 Mon Sep 17 00:00:00 2001 From: Travis Grigsby Date: Wed, 4 May 2022 16:19:59 -0700 Subject: [PATCH 13/56] return fclass_iso, fclass_tlc, and label_x, label_y for OSM features with matching NE features (#2082) --- data/functions.sql | 59 +++++++++++++++++++++++++++++++++ queries/planet_osm_point.jinja2 | 2 ++ 2 files changed, 61 insertions(+) diff --git a/data/functions.sql b/data/functions.sql index 4af7d07fd..80dbf16ad 100644 --- a/data/functions.sql +++ b/data/functions.sql @@ -1053,6 +1053,65 @@ BEGIN END; $$ LANGUAGE plpgsql STABLE; +CREATE OR REPLACE FUNCTION tz_get_fclass_and_label_position(wikidata_id TEXT, place_tag TEXT) +RETURNS JSONB AS $$ +DECLARE +fclass_iso TEXT; +fclass_tlc TEXT; +label_x REAL; +label_y REAL; +BEGIN + IF wikidata_id IS NULL THEN + RETURN '{}'::jsonb; +END IF; + + -- if it's a country, only look it up in the iso and tlc tables + IF place_tag='country' OR place_tag='unrecognized' THEN + SELECT + fclass_iso, fclass_tlc, label_x, label_y INTO fclass_iso, fclass_tlc, label_x, label_y + FROM ne_10m_admin_0_countries_iso i + WHERE i.wikidataid = wikidata_id; + + IF NOT FOUND THEN + SELECT + fclass_iso, fclass_tlc, label_x, label_y INTO fclass_iso, fclass_tlc, label_x, label_y + FROM ne_10m_admin_0_countries_tlc t + WHERE t.wikidataid = wikidata_id; + END IF; + + IF NOT FOUND THEN + RETURN '{}'::jsonb; + END IF; + END IF; + + -- There is no label_x and label_y for the non-countries (please confirm @nkelso!) + SELECT + fclass_iso, fclass_tlc, NULL, NULL INTO fclass_iso, fclass_tlc, label_x, label_y + FROM ne_10m_admin_1_states_provinces sp + WHERE sp.wikidataid = wikidata_id; + + -- finally, try localities + IF NOT FOUND THEN + SELECT + fclass_iso, fclass_tlc, NULL, NULL INTO fclass_iso, fclass_tlc, label_x, label_y + FROM ne_10m_populated_places pp + WHERE pp.wikidataid = wikidata_id;END IF; + + -- return an empty JSONB rather than null, so that it can be safely + -- concatenated with whatever other JSONB rather than needing a check for + -- null. + IF NOT FOUND THEN + RETURN '{}'::jsonb; + END IF; + RETURN jsonb_build_object( + '__ne_fclass_iso', fclass_iso, + '__ne_fclass_tlc', fclass_tlc, + '__ne_label_x', label_x, + '__ne_label_y', label_y + ); +END; +$$ LANGUAGE plpgsql STABLE; + -- returns a JSONB object containing __ne_pop_min and __ne_pop_max CREATE OR REPLACE FUNCTION tz_get_ne_pop_min_max(wikidata_id TEXT) diff --git a/queries/planet_osm_point.jinja2 b/queries/planet_osm_point.jinja2 index 2e9e332ec..4b5e3dc05 100644 --- a/queries/planet_osm_point.jinja2 +++ b/queries/planet_osm_point.jinja2 @@ -30,6 +30,8 @@ SELECT tz_get_ne_min_max_zoom(tags->'wikidata', tags->'place') || tz_get_ne_pop_min_max(tags->'wikidata') + || + tz_get_fclass_and_label_position(tags->'wikidata', tags->'place') END AS __places_properties__, CASE WHEN mz_poi_min_zoom IS NOT NULL From b51e37259269c83230595f2a939dd77aa2f05a25 Mon Sep 17 00:00:00 2001 From: Travis Grigsby Date: Wed, 4 May 2022 16:48:12 -0700 Subject: [PATCH 14/56] adding TLC POV to NE export, YAML files and transform.py (#2081) --- queries/ne.jinja2 | 2 +- vectordatasource/transform.py | 14 +++++++------- yaml/boundaries.yaml | 1 + yaml/places.yaml | 2 ++ 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/queries/ne.jinja2 b/queries/ne.jinja2 index 753ebadf0..f9f49e562 100644 --- a/queries/ne.jinja2 +++ b/queries/ne.jinja2 @@ -1,5 +1,5 @@ {% set ne_languages = ['ar', 'bn', 'de', 'el', 'en', 'es', 'fa', 'fr', 'he', 'hi', 'hu', 'id', 'it', 'ja', 'ko', 'nl', 'pl', 'pt', 'ru', 'sv', 'tr', 'uk', 'ur', 'vi', 'zh', 'zht'] %} -{% set ne_viewpoints = ['iso', 'ar', 'bd', 'br', 'cn', 'de', 'eg', 'es', 'fr', 'gb', 'gr', 'id', 'il', 'in', 'it', 'jp', 'ko', 'ma', 'nl', 'np', 'pk', 'pl', 'ps', 'pt', 'ru', 'sa', 'se', 'tr', 'tw', 'ua', 'us', 'vn'] %} +{% set ne_viewpoints = ['iso', 'tlc', 'ar', 'bd', 'br', 'cn', 'de', 'eg', 'es', 'fr', 'gb', 'gr', 'id', 'il', 'in', 'it', 'jp', 'ko', 'ma', 'nl', 'np', 'pk', 'pl', 'ps', 'pt', 'ru', 'sa', 'se', 'tr', 'tw', 'ua', 'us', 'vn'] %} {% macro ne_common_properties() %} diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index 623d7ce47..bb4ecece1 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -7,6 +7,12 @@ from math import ceil from numbers import Number from sys import float_info +from tilequeue.process import _make_valid_if_necessary +from tilequeue.process import _visible_shape +from tilequeue.tile import calc_meters_per_pixel_area +from tilequeue.tile import normalize_geometry_type +from tilequeue.tile import tolerance_for_zoom +from tilequeue.transform import calculate_padded_bounds import hanzidentifier import kdtree @@ -28,12 +34,6 @@ from shapely.strtree import STRtree from sort import pois as sort_pois from StreetNames import short_street_name -from tilequeue.process import _make_valid_if_necessary -from tilequeue.process import _visible_shape -from tilequeue.tile import calc_meters_per_pixel_area -from tilequeue.tile import normalize_geometry_type -from tilequeue.tile import tolerance_for_zoom -from tilequeue.transform import calculate_padded_bounds from util import safe_int from util import to_float from zope.dottedname.resolve import resolve @@ -9295,7 +9295,7 @@ def _list_of_countries(value): # should have an ISO 3166-1 alpha-2 code, so should be 2 ASCII # latin characters. candidate = candidate.strip().lower() - if candidate == 'iso' or match('[a-z][a-z]', candidate): + if candidate == 'iso' or candidate == 'tlc' or match('[a-z][a-z]', candidate): countries.append(candidate) return countries diff --git a/yaml/boundaries.yaml b/yaml/boundaries.yaml index 6be094f84..e3c89c577 100644 --- a/yaml/boundaries.yaml +++ b/yaml/boundaries.yaml @@ -29,6 +29,7 @@ global: - &ne_localized_kind_properties 'kind:iso': {col: fclass_iso } + 'kind:tlc': {col: fclass_tlc } 'kind:ar': {col: fclass_ar } 'kind:bd': {col: fclass_bd } 'kind:br': {col: fclass_br } diff --git a/yaml/places.yaml b/yaml/places.yaml index 9ab087abd..72323ff3a 100644 --- a/yaml/places.yaml +++ b/yaml/places.yaml @@ -36,6 +36,7 @@ global: wikidata_id: {col: wikidata} disputed_by: {col: disputed_by } 'place:ISO': { col: 'place:ISO' } + 'place:TLC': { col: 'place:TLC' } 'place:AR': { col: 'place:AR' } 'place:BD': { col: 'place:BD' } 'place:BR': { col: 'place:BR' } @@ -83,6 +84,7 @@ global: - &alternate_fclass fclass_iso: { col: fclass_iso } + fclass_tlc: { col: fclass_tlc } fclass_ar: { col: fclass_ar } fclass_bd: { col: fclass_bd } fclass_br: { col: fclass_br } From e0fa3de3d4a4567327f3c3be9659529a4702fe23 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 4 May 2022 23:50:47 +0000 Subject: [PATCH 15/56] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- vectordatasource/transform.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index bb4ecece1..b6dbd8051 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -7,12 +7,6 @@ from math import ceil from numbers import Number from sys import float_info -from tilequeue.process import _make_valid_if_necessary -from tilequeue.process import _visible_shape -from tilequeue.tile import calc_meters_per_pixel_area -from tilequeue.tile import normalize_geometry_type -from tilequeue.tile import tolerance_for_zoom -from tilequeue.transform import calculate_padded_bounds import hanzidentifier import kdtree @@ -34,6 +28,12 @@ from shapely.strtree import STRtree from sort import pois as sort_pois from StreetNames import short_street_name +from tilequeue.process import _make_valid_if_necessary +from tilequeue.process import _visible_shape +from tilequeue.tile import calc_meters_per_pixel_area +from tilequeue.tile import normalize_geometry_type +from tilequeue.tile import tolerance_for_zoom +from tilequeue.transform import calculate_padded_bounds from util import safe_int from util import to_float from zope.dottedname.resolve import resolve From 2135ec4e8e377a956e1c672cc19247208b65e4a6 Mon Sep 17 00:00:00 2001 From: junderwood Date: Thu, 5 May 2022 10:25:48 -0700 Subject: [PATCH 16/56] Sql to recast countries --- data/apple-ne_country_label_recasting.sql | 41 +++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 data/apple-ne_country_label_recasting.sql diff --git a/data/apple-ne_country_label_recasting.sql b/data/apple-ne_country_label_recasting.sql new file mode 100644 index 000000000..3a4e5a528 --- /dev/null +++ b/data/apple-ne_country_label_recasting.sql @@ -0,0 +1,41 @@ +-- This adjusts a few rows in the ne_10m_admin_0_countries_iso and ne_10m_admin_0_countries_tlc tables +-- to either show labels or hide them. + + +-- Akrotiri Sovereign Base Area +update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320741; + +-- Aland +update ne_10m_admin_0_countries_iso set fclass_iso = 'unrecognized' where ne_id = 1159320621; +update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320621; + +-- Antarctica +update ne_10m_admin_0_countries_iso set fclass_iso = 'unrecognized' where ne_id = 1159320335; +update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320335; + +-- Ashmore and Cartier Islands +update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320353; + +-- Clipperton Island +update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320635; + +-- Coral Sea Islands +update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320359; + +-- Dhekelia Cantonment +update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320709; + +-- Gaza Strip +update ne_10m_admin_0_countries_tlc set fclass_tlc = 'Admin-0 country' where ne_id = 1159320901; + +-- Palestine +update ne_10m_admin_0_countries_iso set fclass_iso = 'unrecognized' where ne_id = 1159320899; + +-- Somaliland +update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159321259; + +-- Turkish Republic of Northern Cyprus +update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320531; + +-- West Bank +update ne_10m_admin_0_countries_tlc set fclass_tlc = 'Admin-0 country' where ne_id = 1159320903; \ No newline at end of file From 0004a79805c4b415b5b4cc411eeee284f6d95998 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 20:59:59 +0000 Subject: [PATCH 17/56] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- data/apple-ne_country_label_recasting.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/apple-ne_country_label_recasting.sql b/data/apple-ne_country_label_recasting.sql index 3a4e5a528..5e8122f9b 100644 --- a/data/apple-ne_country_label_recasting.sql +++ b/data/apple-ne_country_label_recasting.sql @@ -38,4 +38,4 @@ update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320531; -- West Bank -update ne_10m_admin_0_countries_tlc set fclass_tlc = 'Admin-0 country' where ne_id = 1159320903; \ No newline at end of file +update ne_10m_admin_0_countries_tlc set fclass_tlc = 'Admin-0 country' where ne_id = 1159320903; From a4b218a4e7d35c4c3e0a704493e6d5ff403059df Mon Sep 17 00:00:00 2001 From: Travis Grigsby Date: Thu, 5 May 2022 14:08:00 -0700 Subject: [PATCH 18/56] updating comment --- data/functions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/functions.sql b/data/functions.sql index 80dbf16ad..233967283 100644 --- a/data/functions.sql +++ b/data/functions.sql @@ -1084,7 +1084,7 @@ END IF; END IF; END IF; - -- There is no label_x and label_y for the non-countries (please confirm @nkelso!) + -- There is no label_x and label_y for the non-countries SELECT fclass_iso, fclass_tlc, NULL, NULL INTO fclass_iso, fclass_tlc, label_x, label_y FROM ne_10m_admin_1_states_provinces sp From 9581f998a96dbe1879256693f3ef859a852d2ff2 Mon Sep 17 00:00:00 2001 From: junderwood Date: Thu, 5 May 2022 14:13:08 -0700 Subject: [PATCH 19/56] Added some filters to country matching --- data/apple-ne_country_label_recasting.sql | 4 ++-- data/functions.sql | 3 ++- data/perform-sql-updates.sh | 5 +++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/data/apple-ne_country_label_recasting.sql b/data/apple-ne_country_label_recasting.sql index 3a4e5a528..9ac792c6b 100644 --- a/data/apple-ne_country_label_recasting.sql +++ b/data/apple-ne_country_label_recasting.sql @@ -1,5 +1,5 @@ --- This adjusts a few rows in the ne_10m_admin_0_countries_iso and ne_10m_admin_0_countries_tlc tables --- to either show labels or hide them. +-- Recasts a few rows in the ne_10m_admin_0_countries_iso and ne_10m_admin_0_countries_tlc +-- tables to either show or hide certain labels -- Akrotiri Sovereign Base Area diff --git a/data/functions.sql b/data/functions.sql index 80dbf16ad..c14a2057c 100644 --- a/data/functions.sql +++ b/data/functions.sql @@ -1076,7 +1076,8 @@ END IF; SELECT fclass_iso, fclass_tlc, label_x, label_y INTO fclass_iso, fclass_tlc, label_x, label_y FROM ne_10m_admin_0_countries_tlc t - WHERE t.wikidataid = wikidata_id; + WHERE t.wikidataid = wikidata_id + AND fclass_tlc IN ('Admin-0 country' or 'Admin-0 dependency'); END IF; IF NOT FOUND THEN diff --git a/data/perform-sql-updates.sh b/data/perform-sql-updates.sh index bbad0d330..9606fc98b 100755 --- a/data/perform-sql-updates.sh +++ b/data/perform-sql-updates.sh @@ -47,6 +47,11 @@ psql $PSQLOPTS $@ -f apply-planet_disputed_area_features_name_suppression.sql echo -e "\nRecasting unwanted NE disputed lines to unrecognized" psql $PSQLOPTS $@ -f apply-ne_disputed_border_suppression.sql +# Recasts a few rows in the ne_10m_admin_0_countries_iso and ne_10m_admin_0_countries_tlc +# tables to either show or hide certain labels. +echo -e "\nRecasting NE country labels" +psql $PSQLOPTS $@ -f apply-ne_country_label_recasting.sql + # Australia suburbs are treated more like cities than typical US style suburbs so we recast them to place=town echo -e "\nRecasting Australia suburbs" psql $PSQLOPTS $@ -f apply-planet_australia_suburb_recast.sql From 1a8a0202c46ceab59d784cc1b47b87e5f2a8bd8c Mon Sep 17 00:00:00 2001 From: Travis Grigsby Date: Thu, 5 May 2022 16:39:52 -0700 Subject: [PATCH 20/56] updating prj for admin_0 countries assets --- data/assets.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/data/assets.yaml b/data/assets.yaml index 683929015..d7d2d8423 100644 --- a/data/assets.yaml +++ b/data/assets.yaml @@ -1,5 +1,5 @@ bucket: nextzen-tile-assets -datestamp: 20220429 +datestamp: 20220505 shapefiles: @@ -141,12 +141,15 @@ shapefiles: - name: ne_10m_admin_0_countries url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_countries.zip + prj: 3857 - name: ne_10m_admin_0_countries_iso url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_countries_iso.zip + prj: 3857 - name: ne_10m_admin_0_countries_tlc url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_countries_tlc.zip + prj: 3857 - name: ne_10m_admin_0_map_units url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_map_units.zip From 9af84a0672393a03aa3f7e2e75d54fa9e950b663 Mon Sep 17 00:00:00 2001 From: junderwood Date: Fri, 6 May 2022 08:54:31 -0700 Subject: [PATCH 21/56] fixed typo in name --- ...y_label_recasting.sql => apply-ne_country_label_recasting.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename data/{apple-ne_country_label_recasting.sql => apply-ne_country_label_recasting.sql} (100%) diff --git a/data/apple-ne_country_label_recasting.sql b/data/apply-ne_country_label_recasting.sql similarity index 100% rename from data/apple-ne_country_label_recasting.sql rename to data/apply-ne_country_label_recasting.sql From 3ba8648a07b82562cd21260797e0c36d83722182 Mon Sep 17 00:00:00 2001 From: Travis Grigsby Date: Fri, 6 May 2022 12:28:11 -0700 Subject: [PATCH 22/56] update .gitignore --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 17e81d1cc..9985e51fa 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,9 @@ test_server.port # files generated during installation of software necessary for testing .eggs/* + +# build artifacts +build/* + +# generated by jetbrains ides +.idea/* From 31a5b4a27b76b89b24214157b4758b3ddebad030 Mon Sep 17 00:00:00 2001 From: Travis Grigsby Date: Fri, 6 May 2022 12:45:33 -0700 Subject: [PATCH 23/56] PR feedback --- data/functions.sql | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/data/functions.sql b/data/functions.sql index 233967283..e6b487e41 100644 --- a/data/functions.sql +++ b/data/functions.sql @@ -1019,27 +1019,22 @@ BEGIN FROM ne_10m_admin_0_countries_tlc t WHERE t.wikidataid = wikidata_id; END IF; - - IF NOT FOUND THEN - RETURN '{}'::jsonb; - END IF; - END IF; - - -- try states and provinces if it's not a country - SELECT - min_label, max_label INTO min_zoom, max_zoom - FROM ne_10m_admin_1_states_provinces sp - WHERE sp.wikidataid = wikidata_id; - - -- finally, try localities - -- There is no concept of max_zoom for ne_10m_populated_places - IF NOT FOUND THEN + ELSE + -- try states and provinces if it's not a country SELECT - pp.min_zoom, NULL INTO min_zoom, max_zoom - FROM ne_10m_populated_places pp - WHERE pp.wikidataid = wikidata_id; - END IF; + min_label, max_label INTO min_zoom, max_zoom + FROM ne_10m_admin_1_states_provinces sp + WHERE sp.wikidataid = wikidata_id; + -- finally, try localities + -- There is no concept of max_zoom for ne_10m_populated_places + IF NOT FOUND THEN + SELECT + pp.min_zoom, NULL INTO min_zoom, max_zoom + FROM ne_10m_populated_places pp + WHERE pp.wikidataid = wikidata_id; + END IF; + END IF; -- return an empty JSONB rather than null, so that it can be safely -- concatenated with whatever other JSONB rather than needing a check for -- null. @@ -1086,14 +1081,14 @@ END IF; -- There is no label_x and label_y for the non-countries SELECT - fclass_iso, fclass_tlc, NULL, NULL INTO fclass_iso, fclass_tlc, label_x, label_y + fclass_iso, fclass_tlc, longitude, latitude INTO fclass_iso, fclass_tlc, label_x, label_y FROM ne_10m_admin_1_states_provinces sp WHERE sp.wikidataid = wikidata_id; -- finally, try localities IF NOT FOUND THEN SELECT - fclass_iso, fclass_tlc, NULL, NULL INTO fclass_iso, fclass_tlc, label_x, label_y + fclass_iso, fclass_tlc, longitude, latitude INTO fclass_iso, fclass_tlc, label_x, label_y FROM ne_10m_populated_places pp WHERE pp.wikidataid = wikidata_id;END IF; From 4bd88709309eb97128e023c4ac32da4216c65506 Mon Sep 17 00:00:00 2001 From: junderwood Date: Fri, 6 May 2022 17:03:06 -0700 Subject: [PATCH 24/56] add back changes --- data/functions.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/functions.sql b/data/functions.sql index ffe2243eb..68c9cf7fc 100644 --- a/data/functions.sql +++ b/data/functions.sql @@ -1017,7 +1017,8 @@ BEGIN SELECT min_label, max_label INTO min_zoom, max_zoom FROM ne_10m_admin_0_countries_tlc t - WHERE t.wikidataid = wikidata_id; + WHERE t.wikidataid = wikidata_id + AND fclass_tlc IN ('Admin-0 country' or 'Admin-0 dependency'); END IF; IF NOT FOUND THEN From 761ae3ca007175e0fd3d9ccb2abe59ae0eaf3027 Mon Sep 17 00:00:00 2001 From: Travis Grigsby Date: Mon, 9 May 2022 10:11:39 -0700 Subject: [PATCH 25/56] adding tests --- integration-test/2081-tlc-pov.py | 102 +++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 integration-test/2081-tlc-pov.py diff --git a/integration-test/2081-tlc-pov.py b/integration-test/2081-tlc-pov.py new file mode 100644 index 000000000..98dea89e1 --- /dev/null +++ b/integration-test/2081-tlc-pov.py @@ -0,0 +1,102 @@ +import dsl + +from . import FixtureTest + + +class TestTLCPOV(FixtureTest): + def test_tlc_ne_place(self): + import dsl + + z, x, y = 16, 0, 0 + + self.generate_fixtures( + dsl.way(1, dsl.tile_centre_shape(z, x, y), { + 'name': 'Foo', + 'featurecla': 'Admin-0 capital', + 'fclass_iso': 'Admin-1 capital', + 'fclass_tlc': 'Admin-0 capital', + 'scalerank': 4, + 'min_zoom': 4, + 'source': 'naturalearthdata.com', + }), + ) + + self.assert_has_feature( + z, x, y, 'places', { + 'kind': 'locality', + 'country_capital': type(True), + 'country_capital:iso': type(False), + 'region_capital:iso': type(True), + 'country_capital:tlc': type(True), + }) + + def test_tlc_ne_boundary(self): + import dsl + + z, x, y = 16, 0, 0 + + self.generate_fixtures( + dsl.way(1, dsl.tile_diagonal(z, x, y), { + 'name': 'Foo', + 'featurecla': 'Admin-1 region boundary', + 'fclass_iso': 'Admin-1 region boundary', + 'fclass_tlc': 'International boundary (verify)', + 'scalerank': 4, + 'min_zoom': 4, + 'source': 'naturalearthdata.com', + }), + ) + + self.assert_has_feature( + z, x, y, 'boundaries', { + 'kind': 'macroregion', + 'kind:iso': 'macroregion', + 'kind:tlc': 'country', + }) + + def test_osm_admin_level_viewpoint_tlc(self): + z, x, y = (16, 39109, 26572) + + self.generate_fixtures( + dsl.way(726514231, dsl.tile_diagonal(z, x, y), { + 'admin_level': '4', + 'admin_level:ISO': '8', + 'admin_level:TLC': '8', + 'boundary': 'disputed', + 'name': 'Viewpoints on Disputed Administrative Boundaries', + 'type': 'linestring', + 'source': 'openstreetmap.org', + }), + ) + + self.assert_has_feature( + z, x, y, 'boundaries', { + 'id': 726514231, + 'kind': 'disputed_reference_line', + 'kind:iso': 'locality', + 'kind:tlc': 'locality', + }) + + def test_osm_places_with_viewpoint_tlc(self): + import dsl + + z, x, y = (10, 856, 441) + + self.generate_fixtures( + dsl.point(432425099, (120.9820179, 23.9739374), { + 'name': 'Test place', + 'place': 'country', + 'place:ISO': 'state', + 'place:TLC': 'district', + 'source': 'openstreetmap.org', + 'source:sqkm': 'CIA World Factbook', + }), + ) + + self.assert_has_feature( + z, x, y, 'places', { + 'id': 432425099, + 'kind': 'country', + 'kind:iso': 'region', + 'kind:tlc': 'county' + }) From 2635448740ded3a712fbb9cc5c61d108ac734ec9 Mon Sep 17 00:00:00 2001 From: Travis Grigsby Date: Mon, 9 May 2022 12:43:29 -0700 Subject: [PATCH 26/56] updating assets file to 5.1.1, removing projection spec --- data/assets.yaml | 71 +++++++++++++++++++++++------------------------- 1 file changed, 34 insertions(+), 37 deletions(-) diff --git a/data/assets.yaml b/data/assets.yaml index d7d2d8423..66fd1d5cf 100644 --- a/data/assets.yaml +++ b/data/assets.yaml @@ -1,5 +1,5 @@ bucket: nextzen-tile-assets -datestamp: 20220505 +datestamp: 20220509 shapefiles: @@ -55,107 +55,104 @@ shapefiles: url: http://s3.amazonaws.com/tilezen-assets/curated/admin_areas_20180409.zip - name: ne_110m_lakes - url: https://naciscdn.org/naturalearth/5.1.0/110m/physical/ne_110m_lakes.zip + url: https://naciscdn.org/naturalearth/5.1.1/110m/physical/ne_110m_lakes.zip - name: ne_50m_lakes - url: https://naciscdn.org/naturalearth/5.1.0/50m/physical/ne_50m_lakes.zip + url: https://naciscdn.org/naturalearth/5.1.1/50m/physical/ne_50m_lakes.zip - name: ne_10m_lakes - url: https://naciscdn.org/naturalearth/5.1.0/10m/physical/ne_10m_lakes.zip + url: https://naciscdn.org/naturalearth/5.1.1/10m/physical/ne_10m_lakes.zip - name: ne_110m_ocean - url: https://naciscdn.org/naturalearth/5.1.0/110m/physical/ne_110m_ocean.zip + url: https://naciscdn.org/naturalearth/5.1.1/110m/physical/ne_110m_ocean.zip - name: ne_50m_ocean - url: https://naciscdn.org/naturalearth/5.1.0/50m/physical/ne_50m_ocean.zip + url: https://naciscdn.org/naturalearth/5.1.1/50m/physical/ne_50m_ocean.zip - name: ne_10m_ocean - url: https://naciscdn.org/naturalearth/5.1.0/10m/physical/ne_10m_ocean.zip + url: https://naciscdn.org/naturalearth/5.1.1/10m/physical/ne_10m_ocean.zip - name: ne_50m_playas - url: https://naciscdn.org/naturalearth/5.1.0/50m/physical/ne_50m_playas.zip + url: https://naciscdn.org/naturalearth/5.1.1/50m/physical/ne_50m_playas.zip - name: ne_10m_playas - url: https://naciscdn.org/naturalearth/5.1.0/10m/physical/ne_10m_playas.zip + url: https://naciscdn.org/naturalearth/5.1.1/10m/physical/ne_10m_playas.zip - name: ne_50m_urban_areas - url: https://naciscdn.org/naturalearth/5.1.0/50m/cultural/ne_50m_urban_areas.zip + url: https://naciscdn.org/naturalearth/5.1.1/50m/cultural/ne_50m_urban_areas.zip - name: ne_10m_urban_areas - url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_urban_areas.zip + url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_urban_areas.zip - name: ne_110m_land - url: https://naciscdn.org/naturalearth/5.1.0/110m/physical/ne_110m_land.zip + url: https://naciscdn.org/naturalearth/5.1.1/110m/physical/ne_110m_land.zip - name: ne_50m_land - url: https://naciscdn.org/naturalearth/5.1.0/50m/physical/ne_50m_land.zip + url: https://naciscdn.org/naturalearth/5.1.1/50m/physical/ne_50m_land.zip - name: ne_10m_land - url: https://naciscdn.org/naturalearth/5.1.0/10m/physical/ne_10m_land.zip + url: https://naciscdn.org/naturalearth/5.1.1/10m/physical/ne_10m_land.zip tile: true - name: ne_10m_populated_places - url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_populated_places.zip + url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_populated_places.zip - name: ne_110m_admin_0_boundary_lines_land - url: https://naciscdn.org/naturalearth/5.1.0/110m/cultural/ne_110m_admin_0_boundary_lines_land.zip + url: https://naciscdn.org/naturalearth/5.1.1/110m/cultural/ne_110m_admin_0_boundary_lines_land.zip - name: ne_50m_admin_0_boundary_lines_land - url: https://naciscdn.org/naturalearth/5.1.0/50m/cultural/ne_50m_admin_0_boundary_lines_land.zip + url: https://naciscdn.org/naturalearth/5.1.1/50m/cultural/ne_50m_admin_0_boundary_lines_land.zip - name: ne_10m_admin_0_boundary_lines_land - url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_boundary_lines_land.zip + url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_0_boundary_lines_land.zip - name: ne_10m_admin_0_boundary_lines_map_units - url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_boundary_lines_map_units.zip + url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_0_boundary_lines_map_units.zip - name: ne_50m_admin_1_states_provinces_lines - url: https://naciscdn.org/naturalearth/5.1.0/50m/cultural/ne_50m_admin_1_states_provinces_lines.zip + url: https://naciscdn.org/naturalearth/5.1.1/50m/cultural/ne_50m_admin_1_states_provinces_lines.zip - name: ne_10m_admin_1_states_provinces_lines - url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_1_states_provinces_lines.zip + url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_1_states_provinces_lines.zip - name: ne_50m_admin_0_boundary_lines_disputed_areas - url: https://naciscdn.org/naturalearth/5.1.0/50m/cultural/ne_50m_admin_0_boundary_lines_disputed_areas.zip + url: https://naciscdn.org/naturalearth/5.1.1/50m/cultural/ne_50m_admin_0_boundary_lines_disputed_areas.zip - name: ne_50m_admin_0_boundary_lines_maritime_indicator_chn - url: https://naciscdn.org/naturalearth/5.1.0/50m/cultural/ne_50m_admin_0_boundary_lines_maritime_indicator_chn.zip + url: https://naciscdn.org/naturalearth/5.1.1/50m/cultural/ne_50m_admin_0_boundary_lines_maritime_indicator_chn.zip - name: ne_10m_admin_0_boundary_lines_disputed_areas - url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_boundary_lines_disputed_areas.zip + url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_0_boundary_lines_disputed_areas.zip - name: ne_10m_admin_0_boundary_lines_maritime_indicator_chn - url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_boundary_lines_maritime_indicator_chn.zip + url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_0_boundary_lines_maritime_indicator_chn.zip - name: ne_10m_roads - url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_roads.zip + url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_roads.zip - name: ne_110m_coastline - url: https://naciscdn.org/naturalearth/5.1.0/110m/physical/ne_110m_coastline.zip + url: https://naciscdn.org/naturalearth/5.1.1/110m/physical/ne_110m_coastline.zip - name: ne_50m_coastline - url: https://naciscdn.org/naturalearth/5.1.0/50m/physical/ne_50m_coastline.zip + url: https://naciscdn.org/naturalearth/5.1.1/50m/physical/ne_50m_coastline.zip - name: ne_10m_coastline - url: https://naciscdn.org/naturalearth/5.1.0/10m/physical/ne_10m_coastline.zip + url: https://naciscdn.org/naturalearth/5.1.1/10m/physical/ne_10m_coastline.zip - name: ne_10m_admin_0_countries - url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_countries.zip - prj: 3857 + url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_0_countries.zip - name: ne_10m_admin_0_countries_iso - url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_countries_iso.zip - prj: 3857 + url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_0_countries_iso.zip - name: ne_10m_admin_0_countries_tlc - url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_countries_tlc.zip - prj: 3857 + url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_0_countries_tlc.zip - name: ne_10m_admin_0_map_units - url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_map_units.zip + url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_0_map_units.zip - name: ne_10m_admin_1_states_provinces - url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_1_states_provinces.zip + url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_1_states_provinces.zip wikidata-queries: - name: aerodrome_passenger_count From 2e59334a5aa3b6cfe032b6152399cb1246600d95 Mon Sep 17 00:00:00 2001 From: Travis Grigsby Date: Wed, 11 May 2022 14:44:37 -0600 Subject: [PATCH 27/56] disambiguate --- data/functions.sql | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/data/functions.sql b/data/functions.sql index e6b487e41..4aba7bcb9 100644 --- a/data/functions.sql +++ b/data/functions.sql @@ -1051,10 +1051,10 @@ $$ LANGUAGE plpgsql STABLE; CREATE OR REPLACE FUNCTION tz_get_fclass_and_label_position(wikidata_id TEXT, place_tag TEXT) RETURNS JSONB AS $$ DECLARE -fclass_iso TEXT; -fclass_tlc TEXT; -label_x REAL; -label_y REAL; +fclass_iso_var TEXT; +fclass_tlc_var TEXT; +label_x_var REAL; +label_y_var REAL; BEGIN IF wikidata_id IS NULL THEN RETURN '{}'::jsonb; @@ -1063,13 +1063,13 @@ END IF; -- if it's a country, only look it up in the iso and tlc tables IF place_tag='country' OR place_tag='unrecognized' THEN SELECT - fclass_iso, fclass_tlc, label_x, label_y INTO fclass_iso, fclass_tlc, label_x, label_y + fclass_iso, fclass_tlc, label_x, label_y INTO fclass_iso_var, fclass_tlc_var, label_x_var, label_y_var FROM ne_10m_admin_0_countries_iso i WHERE i.wikidataid = wikidata_id; IF NOT FOUND THEN SELECT - fclass_iso, fclass_tlc, label_x, label_y INTO fclass_iso, fclass_tlc, label_x, label_y + fclass_iso, fclass_tlc, label_x, label_y INTO fclass_iso_var, fclass_tlc_var, label_x_var, label_y_var FROM ne_10m_admin_0_countries_tlc t WHERE t.wikidataid = wikidata_id; END IF; @@ -1081,14 +1081,14 @@ END IF; -- There is no label_x and label_y for the non-countries SELECT - fclass_iso, fclass_tlc, longitude, latitude INTO fclass_iso, fclass_tlc, label_x, label_y + fclass_iso, fclass_tlc, longitude, latitude INTO fclass_iso_var, fclass_tlc_var, label_x_var, label_y_var FROM ne_10m_admin_1_states_provinces sp WHERE sp.wikidataid = wikidata_id; -- finally, try localities IF NOT FOUND THEN SELECT - fclass_iso, fclass_tlc, longitude, latitude INTO fclass_iso, fclass_tlc, label_x, label_y + fclass_iso, fclass_tlc, longitude, latitude INTO fclass_iso_var, fclass_tlc_var, label_x_var, label_y_var FROM ne_10m_populated_places pp WHERE pp.wikidataid = wikidata_id;END IF; @@ -1099,10 +1099,10 @@ END IF; RETURN '{}'::jsonb; END IF; RETURN jsonb_build_object( - '__ne_fclass_iso', fclass_iso, - '__ne_fclass_tlc', fclass_tlc, - '__ne_label_x', label_x, - '__ne_label_y', label_y + '__ne_fclass_iso', fclass_iso_var, + '__ne_fclass_tlc', fclass_tlc_var, + '__ne_label_x', label_x_var, + '__ne_label_y', label_y_var ); END; $$ LANGUAGE plpgsql STABLE; From 1185427d28d53408fe1b7226e5d3c9e477e453a4 Mon Sep 17 00:00:00 2001 From: Travis Grigsby Date: Fri, 13 May 2022 11:23:01 -0600 Subject: [PATCH 28/56] adding disadmbiguation --- data/functions.sql | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/data/functions.sql b/data/functions.sql index 4aba7bcb9..d32d1746f 100644 --- a/data/functions.sql +++ b/data/functions.sql @@ -1063,13 +1063,13 @@ END IF; -- if it's a country, only look it up in the iso and tlc tables IF place_tag='country' OR place_tag='unrecognized' THEN SELECT - fclass_iso, fclass_tlc, label_x, label_y INTO fclass_iso_var, fclass_tlc_var, label_x_var, label_y_var + i.fclass_iso, i.fclass_tlc, i.label_x, i.label_y INTO fclass_iso_var, fclass_tlc_var, label_x_var, label_y_var FROM ne_10m_admin_0_countries_iso i WHERE i.wikidataid = wikidata_id; IF NOT FOUND THEN SELECT - fclass_iso, fclass_tlc, label_x, label_y INTO fclass_iso_var, fclass_tlc_var, label_x_var, label_y_var + t.fclass_iso, t.fclass_tlc, t.label_x, t.label_y INTO fclass_iso_var, fclass_tlc_var, label_x_var, label_y_var FROM ne_10m_admin_0_countries_tlc t WHERE t.wikidataid = wikidata_id; END IF; @@ -1077,20 +1077,26 @@ END IF; IF NOT FOUND THEN RETURN '{}'::jsonb; END IF; + RETURN jsonb_build_object( + '__ne_fclass_iso', fclass_iso_var, + '__ne_fclass_tlc', fclass_tlc_var, + '__ne_label_x', label_x_var, + '__ne_label_y', label_y_var + ); END IF; - -- There is no label_x and label_y for the non-countries SELECT - fclass_iso, fclass_tlc, longitude, latitude INTO fclass_iso_var, fclass_tlc_var, label_x_var, label_y_var + sp.fclass_iso, sp.fclass_tlc, sp.longitude, sp.latitude INTO fclass_iso_var, fclass_tlc_var, label_x_var, label_y_var FROM ne_10m_admin_1_states_provinces sp WHERE sp.wikidataid = wikidata_id; -- finally, try localities IF NOT FOUND THEN - SELECT - fclass_iso, fclass_tlc, longitude, latitude INTO fclass_iso_var, fclass_tlc_var, label_x_var, label_y_var - FROM ne_10m_populated_places pp - WHERE pp.wikidataid = wikidata_id;END IF; + SELECT + pp.fclass_iso, pp.fclass_tlc, pp.longitude, pp.latitude INTO fclass_iso_var, fclass_tlc_var, label_x_var, label_y_var + FROM ne_10m_populated_places pp + WHERE pp.wikidataid = wikidata_id; + END IF; -- return an empty JSONB rather than null, so that it can be safely -- concatenated with whatever other JSONB rather than needing a check for From 24c86c900b60bc58658bc40e6ad2b3a4d4e4869d Mon Sep 17 00:00:00 2001 From: Travis Grigsby Date: Fri, 13 May 2022 16:10:46 -0700 Subject: [PATCH 29/56] upgrading to 5.1.2 --- data/assets.yaml | 68 ++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/data/assets.yaml b/data/assets.yaml index 66fd1d5cf..f6ff13ab8 100644 --- a/data/assets.yaml +++ b/data/assets.yaml @@ -1,5 +1,5 @@ bucket: nextzen-tile-assets -datestamp: 20220509 +datestamp: 20220513 shapefiles: @@ -55,104 +55,104 @@ shapefiles: url: http://s3.amazonaws.com/tilezen-assets/curated/admin_areas_20180409.zip - name: ne_110m_lakes - url: https://naciscdn.org/naturalearth/5.1.1/110m/physical/ne_110m_lakes.zip + url: https://naciscdn.org/naturalearth/5.1.2/110m/physical/ne_110m_lakes.zip - name: ne_50m_lakes - url: https://naciscdn.org/naturalearth/5.1.1/50m/physical/ne_50m_lakes.zip + url: https://naciscdn.org/naturalearth/5.1.2/50m/physical/ne_50m_lakes.zip - name: ne_10m_lakes - url: https://naciscdn.org/naturalearth/5.1.1/10m/physical/ne_10m_lakes.zip + url: https://naciscdn.org/naturalearth/5.1.2/10m/physical/ne_10m_lakes.zip - name: ne_110m_ocean - url: https://naciscdn.org/naturalearth/5.1.1/110m/physical/ne_110m_ocean.zip + url: https://naciscdn.org/naturalearth/5.1.2/110m/physical/ne_110m_ocean.zip - name: ne_50m_ocean - url: https://naciscdn.org/naturalearth/5.1.1/50m/physical/ne_50m_ocean.zip + url: https://naciscdn.org/naturalearth/5.1.2/50m/physical/ne_50m_ocean.zip - name: ne_10m_ocean - url: https://naciscdn.org/naturalearth/5.1.1/10m/physical/ne_10m_ocean.zip + url: https://naciscdn.org/naturalearth/5.1.2/10m/physical/ne_10m_ocean.zip - name: ne_50m_playas - url: https://naciscdn.org/naturalearth/5.1.1/50m/physical/ne_50m_playas.zip + url: https://naciscdn.org/naturalearth/5.1.2/50m/physical/ne_50m_playas.zip - name: ne_10m_playas - url: https://naciscdn.org/naturalearth/5.1.1/10m/physical/ne_10m_playas.zip + url: https://naciscdn.org/naturalearth/5.1.2/10m/physical/ne_10m_playas.zip - name: ne_50m_urban_areas - url: https://naciscdn.org/naturalearth/5.1.1/50m/cultural/ne_50m_urban_areas.zip + url: https://naciscdn.org/naturalearth/5.1.2/50m/cultural/ne_50m_urban_areas.zip - name: ne_10m_urban_areas - url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_urban_areas.zip + url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_urban_areas.zip - name: ne_110m_land - url: https://naciscdn.org/naturalearth/5.1.1/110m/physical/ne_110m_land.zip + url: https://naciscdn.org/naturalearth/5.1.2/110m/physical/ne_110m_land.zip - name: ne_50m_land - url: https://naciscdn.org/naturalearth/5.1.1/50m/physical/ne_50m_land.zip + url: https://naciscdn.org/naturalearth/5.1.2/50m/physical/ne_50m_land.zip - name: ne_10m_land - url: https://naciscdn.org/naturalearth/5.1.1/10m/physical/ne_10m_land.zip + url: https://naciscdn.org/naturalearth/5.1.2/10m/physical/ne_10m_land.zip tile: true - name: ne_10m_populated_places - url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_populated_places.zip + url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_populated_places.zip - name: ne_110m_admin_0_boundary_lines_land - url: https://naciscdn.org/naturalearth/5.1.1/110m/cultural/ne_110m_admin_0_boundary_lines_land.zip + url: https://naciscdn.org/naturalearth/5.1.2/110m/cultural/ne_110m_admin_0_boundary_lines_land.zip - name: ne_50m_admin_0_boundary_lines_land - url: https://naciscdn.org/naturalearth/5.1.1/50m/cultural/ne_50m_admin_0_boundary_lines_land.zip + url: https://naciscdn.org/naturalearth/5.1.2/50m/cultural/ne_50m_admin_0_boundary_lines_land.zip - name: ne_10m_admin_0_boundary_lines_land - url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_0_boundary_lines_land.zip + url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_0_boundary_lines_land.zip - name: ne_10m_admin_0_boundary_lines_map_units - url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_0_boundary_lines_map_units.zip + url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_0_boundary_lines_map_units.zip - name: ne_50m_admin_1_states_provinces_lines - url: https://naciscdn.org/naturalearth/5.1.1/50m/cultural/ne_50m_admin_1_states_provinces_lines.zip + url: https://naciscdn.org/naturalearth/5.1.2/50m/cultural/ne_50m_admin_1_states_provinces_lines.zip - name: ne_10m_admin_1_states_provinces_lines - url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_1_states_provinces_lines.zip + url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_1_states_provinces_lines.zip - name: ne_50m_admin_0_boundary_lines_disputed_areas - url: https://naciscdn.org/naturalearth/5.1.1/50m/cultural/ne_50m_admin_0_boundary_lines_disputed_areas.zip + url: https://naciscdn.org/naturalearth/5.1.2/50m/cultural/ne_50m_admin_0_boundary_lines_disputed_areas.zip - name: ne_50m_admin_0_boundary_lines_maritime_indicator_chn - url: https://naciscdn.org/naturalearth/5.1.1/50m/cultural/ne_50m_admin_0_boundary_lines_maritime_indicator_chn.zip + url: https://naciscdn.org/naturalearth/5.1.2/50m/cultural/ne_50m_admin_0_boundary_lines_maritime_indicator_chn.zip - name: ne_10m_admin_0_boundary_lines_disputed_areas - url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_0_boundary_lines_disputed_areas.zip + url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_0_boundary_lines_disputed_areas.zip - name: ne_10m_admin_0_boundary_lines_maritime_indicator_chn - url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_0_boundary_lines_maritime_indicator_chn.zip + url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_0_boundary_lines_maritime_indicator_chn.zip - name: ne_10m_roads - url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_roads.zip + url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_roads.zip - name: ne_110m_coastline - url: https://naciscdn.org/naturalearth/5.1.1/110m/physical/ne_110m_coastline.zip + url: https://naciscdn.org/naturalearth/5.1.2/110m/physical/ne_110m_coastline.zip - name: ne_50m_coastline - url: https://naciscdn.org/naturalearth/5.1.1/50m/physical/ne_50m_coastline.zip + url: https://naciscdn.org/naturalearth/5.1.2/50m/physical/ne_50m_coastline.zip - name: ne_10m_coastline - url: https://naciscdn.org/naturalearth/5.1.1/10m/physical/ne_10m_coastline.zip + url: https://naciscdn.org/naturalearth/5.1.2/10m/physical/ne_10m_coastline.zip - name: ne_10m_admin_0_countries - url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_0_countries.zip + url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_0_countries.zip - name: ne_10m_admin_0_countries_iso - url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_0_countries_iso.zip + url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_0_countries_iso.zip - name: ne_10m_admin_0_countries_tlc - url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_0_countries_tlc.zip + url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_0_countries_tlc.zip - name: ne_10m_admin_0_map_units - url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_0_map_units.zip + url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_0_map_units.zip - name: ne_10m_admin_1_states_provinces - url: https://naciscdn.org/naturalearth/5.1.1/10m/cultural/ne_10m_admin_1_states_provinces.zip + url: https://naciscdn.org/naturalearth/5.1.2/10m/cultural/ne_10m_admin_1_states_provinces.zip wikidata-queries: - name: aerodrome_passenger_count From 8fb9226aebabf029de24aa8a2f95fe0dda884c83 Mon Sep 17 00:00:00 2001 From: Travis Grigsby Date: Mon, 16 May 2022 10:04:41 -0700 Subject: [PATCH 30/56] updating shapefile again --- data/assets.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/assets.yaml b/data/assets.yaml index f6ff13ab8..b789fad52 100644 --- a/data/assets.yaml +++ b/data/assets.yaml @@ -1,5 +1,5 @@ bucket: nextzen-tile-assets -datestamp: 20220513 +datestamp: 20220516 shapefiles: From c126fe23aee35c8369d073fea681ccb74c83f00e Mon Sep 17 00:00:00 2001 From: junderwood Date: Mon, 16 May 2022 12:50:09 -0700 Subject: [PATCH 31/56] fixed typo --- data/functions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/functions.sql b/data/functions.sql index 84f750963..a3b0cf4ad 100644 --- a/data/functions.sql +++ b/data/functions.sql @@ -1018,7 +1018,7 @@ BEGIN min_label, max_label INTO min_zoom, max_zoom FROM ne_10m_admin_0_countries_tlc t WHERE t.wikidataid = wikidata_id - AND fclass_tlc IN ('Admin-0 country' or 'Admin-0 dependency'); + AND fclass_tlc IN ('Admin-0 country', 'Admin-0 dependency'); END IF; ELSE -- try states and provinces if it's not a country From de0152f61aba2b54114df0528198c5adc15a8feb Mon Sep 17 00:00:00 2001 From: junderwood Date: Mon, 16 May 2022 14:09:43 -0700 Subject: [PATCH 32/56] fixed typo --- data/functions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/functions.sql b/data/functions.sql index a3b0cf4ad..4a09b09d9 100644 --- a/data/functions.sql +++ b/data/functions.sql @@ -1073,7 +1073,7 @@ END IF; t.fclass_iso, t.fclass_tlc, t.label_x, t.label_y INTO fclass_iso_var, fclass_tlc_var, label_x_var, label_y_var FROM ne_10m_admin_0_countries_tlc t WHERE t.wikidataid = wikidata_id - AND fclass_tlc IN ('Admin-0 country' or 'Admin-0 dependency'); + AND fclass_tlc IN ('Admin-0 country', 'Admin-0 dependency'); END IF; IF NOT FOUND THEN From f03e1394139ff20cf558756a0ace32ddea1df031 Mon Sep 17 00:00:00 2001 From: junderwood Date: Tue, 17 May 2022 09:19:41 -0700 Subject: [PATCH 33/56] Change to use featurecla for comparison. Update country min zoom with case statement --- data/apply-ne_country_label_recasting.sql | 35 ++++++++++------------- data/functions.sql | 4 +-- yaml/places.yaml | 6 +++- 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/data/apply-ne_country_label_recasting.sql b/data/apply-ne_country_label_recasting.sql index 914da08d0..32e447152 100644 --- a/data/apply-ne_country_label_recasting.sql +++ b/data/apply-ne_country_label_recasting.sql @@ -1,41 +1,36 @@ -- Recasts a few rows in the ne_10m_admin_0_countries_iso and ne_10m_admin_0_countries_tlc -- tables to either show or hide certain labels +-- sets featurcla to equal fclass_tlc values. We can then further modify the values while keeping the fclass intact. +update ne_10m_admin_0_countries_tlc set featurecla = fclass_tlc; -- Akrotiri Sovereign Base Area -update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320741; - --- Aland -update ne_10m_admin_0_countries_iso set fclass_iso = 'unrecognized' where ne_id = 1159320621; -update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320621; - --- Antarctica -update ne_10m_admin_0_countries_iso set fclass_iso = 'unrecognized' where ne_id = 1159320335; -update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320335; +update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159320741; -- Ashmore and Cartier Islands -update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320353; +update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159320353; -- Clipperton Island -update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320635; +update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159320635; -- Coral Sea Islands -update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320359; +update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159320359; -- Dhekelia Cantonment -update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320709; +update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159320709; -- Gaza Strip -update ne_10m_admin_0_countries_tlc set fclass_tlc = 'Admin-0 country' where ne_id = 1159320901; - --- Palestine -update ne_10m_admin_0_countries_iso set fclass_iso = 'unrecognized' where ne_id = 1159320899; +update ne_10m_admin_0_countries_tlc set featurecla = 'Admin-0 country' where ne_id = 1159320901; -- Somaliland -update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159321259; +update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159321259; -- Turkish Republic of Northern Cyprus -update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320531; +update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159320531; -- West Bank -update ne_10m_admin_0_countries_tlc set fclass_tlc = 'Admin-0 country' where ne_id = 1159320903; +update ne_10m_admin_0_countries_tlc set featurecla = 'Admin-0 country' where ne_id = 1159320903; + + +-- Fix Saint Helena wikidata id for this build. Remove once NE is updated +update ne_10m_admin_0_countries_iso set wikidata = 'Q192184' where ne_id = 1159320733 \ No newline at end of file diff --git a/data/functions.sql b/data/functions.sql index 4a09b09d9..f2bc6043f 100644 --- a/data/functions.sql +++ b/data/functions.sql @@ -1018,7 +1018,7 @@ BEGIN min_label, max_label INTO min_zoom, max_zoom FROM ne_10m_admin_0_countries_tlc t WHERE t.wikidataid = wikidata_id - AND fclass_tlc IN ('Admin-0 country', 'Admin-0 dependency'); + AND featurecla IN ('Admin-0 country', 'Admin-0 dependency'); END IF; ELSE -- try states and provinces if it's not a country @@ -1073,7 +1073,7 @@ END IF; t.fclass_iso, t.fclass_tlc, t.label_x, t.label_y INTO fclass_iso_var, fclass_tlc_var, label_x_var, label_y_var FROM ne_10m_admin_0_countries_tlc t WHERE t.wikidataid = wikidata_id - AND fclass_tlc IN ('Admin-0 country', 'Admin-0 dependency'); + AND featurecla IN ('Admin-0 country', 'Admin-0 dependency'); END IF; IF NOT FOUND THEN diff --git a/yaml/places.yaml b/yaml/places.yaml index 72323ff3a..c62ff49d4 100644 --- a/yaml/places.yaml +++ b/yaml/places.yaml @@ -147,7 +147,11 @@ filters: # # note: also, we knock this down to 6 during post-processing for any country # which doesn't match an NE curated one. - min_zoom: 1 + min_zoom: + case: + - when: { place: country } + then: 1 + - else: 8 output: <<: [*output_properties, *ne_zooms] kind: {col: place} From 9d59e59e310ffd806478526027197ebd8be5bb5e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 May 2022 16:23:12 +0000 Subject: [PATCH 34/56] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- data/apply-ne_country_label_recasting.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/apply-ne_country_label_recasting.sql b/data/apply-ne_country_label_recasting.sql index 32e447152..5fb9c8c31 100644 --- a/data/apply-ne_country_label_recasting.sql +++ b/data/apply-ne_country_label_recasting.sql @@ -33,4 +33,4 @@ update ne_10m_admin_0_countries_tlc set featurecla = 'Admin-0 country' where ne_ -- Fix Saint Helena wikidata id for this build. Remove once NE is updated -update ne_10m_admin_0_countries_iso set wikidata = 'Q192184' where ne_id = 1159320733 \ No newline at end of file +update ne_10m_admin_0_countries_iso set wikidata = 'Q192184' where ne_id = 1159320733 From 3801e7697dad82c66b375c57c9a9d4a673dfd96a Mon Sep 17 00:00:00 2001 From: junderwood Date: Tue, 17 May 2022 09:23:51 -0700 Subject: [PATCH 35/56] update test --- integration-test/1826-unrecognised-country-labels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/1826-unrecognised-country-labels.py b/integration-test/1826-unrecognised-country-labels.py index c338e4f4a..1b37d5550 100644 --- a/integration-test/1826-unrecognised-country-labels.py +++ b/integration-test/1826-unrecognised-country-labels.py @@ -38,7 +38,7 @@ def test_country_no_ne(self): self.assert_has_feature( z, x, y, 'places', { 'kind': 'unrecognized', - 'min_zoom': 6, + 'min_zoom': 8, }) def test_region_no_ne(self): From 253990146a8d485e82236a566b7d7bfd4cdb3bf5 Mon Sep 17 00:00:00 2001 From: junderwood Date: Tue, 17 May 2022 10:44:35 -0700 Subject: [PATCH 36/56] fixed typo in lua --- osm2pgsql.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osm2pgsql.lua b/osm2pgsql.lua index 6d54c2354..15aac658f 100644 --- a/osm2pgsql.lua +++ b/osm2pgsql.lua @@ -557,7 +557,7 @@ function osm2pgsql.process_node(object) output_hstore['place'] = 'country' end -- New Caledonia - if object.tags.place == 'archipelago' and object.tags['ISO3166-1'] == 'NF' then + if object.tags.place == 'archipelago' and object.tags['ISO3166-1'] == 'NC' then output_hstore['place'] = 'country' end -- Norfolk Island From 2344be281046b7a999da82c1c9f47cb3db61019d Mon Sep 17 00:00:00 2001 From: junderwood Date: Tue, 17 May 2022 10:57:01 -0700 Subject: [PATCH 37/56] Need to alter column length --- data/apply-ne_country_label_recasting.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data/apply-ne_country_label_recasting.sql b/data/apply-ne_country_label_recasting.sql index 5fb9c8c31..0786c18a3 100644 --- a/data/apply-ne_country_label_recasting.sql +++ b/data/apply-ne_country_label_recasting.sql @@ -2,6 +2,8 @@ -- tables to either show or hide certain labels -- sets featurcla to equal fclass_tlc values. We can then further modify the values while keeping the fclass intact. +-- featurecla has a shorter varchar length and needs altering first +alter table ne_10m_admin_0_countries_tlc alter column featurecla type varchar; update ne_10m_admin_0_countries_tlc set featurecla = fclass_tlc; -- Akrotiri Sovereign Base Area From a23f6d14805b619ef45bd9ff65ad292d660433af Mon Sep 17 00:00:00 2001 From: junderwood Date: Tue, 17 May 2022 12:04:42 -0700 Subject: [PATCH 38/56] wrong column name --- data/apply-ne_country_label_recasting.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/apply-ne_country_label_recasting.sql b/data/apply-ne_country_label_recasting.sql index 0786c18a3..08266f159 100644 --- a/data/apply-ne_country_label_recasting.sql +++ b/data/apply-ne_country_label_recasting.sql @@ -35,4 +35,4 @@ update ne_10m_admin_0_countries_tlc set featurecla = 'Admin-0 country' where ne_ -- Fix Saint Helena wikidata id for this build. Remove once NE is updated -update ne_10m_admin_0_countries_iso set wikidata = 'Q192184' where ne_id = 1159320733 +update ne_10m_admin_0_countries_iso set wikidataid = 'Q192184' where ne_id = 1159320733 From 4985dffa29361be4cd57c01652a17f5943ff97b3 Mon Sep 17 00:00:00 2001 From: junderwood Date: Tue, 17 May 2022 12:06:39 -0700 Subject: [PATCH 39/56] add semicolon for neatness --- data/apply-ne_country_label_recasting.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/apply-ne_country_label_recasting.sql b/data/apply-ne_country_label_recasting.sql index 08266f159..50de51b9f 100644 --- a/data/apply-ne_country_label_recasting.sql +++ b/data/apply-ne_country_label_recasting.sql @@ -35,4 +35,4 @@ update ne_10m_admin_0_countries_tlc set featurecla = 'Admin-0 country' where ne_ -- Fix Saint Helena wikidata id for this build. Remove once NE is updated -update ne_10m_admin_0_countries_iso set wikidataid = 'Q192184' where ne_id = 1159320733 +update ne_10m_admin_0_countries_iso set wikidataid = 'Q192184' where ne_id = 1159320733; From 09f1317ba18280c056c99232ad9c6720b3b574de Mon Sep 17 00:00:00 2001 From: junderwood Date: Wed, 18 May 2022 10:46:51 -0700 Subject: [PATCH 40/56] Comment fixes --- data/apply-ne_country_label_recasting.sql | 9 ++++++--- vectordatasource/transform.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/data/apply-ne_country_label_recasting.sql b/data/apply-ne_country_label_recasting.sql index 50de51b9f..217b1b06e 100644 --- a/data/apply-ne_country_label_recasting.sql +++ b/data/apply-ne_country_label_recasting.sql @@ -1,8 +1,11 @@ -- Recasts a few rows in the ne_10m_admin_0_countries_iso and ne_10m_admin_0_countries_tlc --- tables to either show or hide certain labels +-- tables to either show or hide certain labels. +-- The TLC table yields around 10 country and dependency features over ISO (including Kosovo and Taiwan), +-- which Tilezen sometimes has opinions about. + -- sets featurcla to equal fclass_tlc values. We can then further modify the values while keeping the fclass intact. --- featurecla has a shorter varchar length and needs altering first +-- featurecla column has a shorter varchar length and needs altering first. alter table ne_10m_admin_0_countries_tlc alter column featurecla type varchar; update ne_10m_admin_0_countries_tlc set featurecla = fclass_tlc; @@ -31,7 +34,7 @@ update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159320531; -- West Bank -update ne_10m_admin_0_countries_tlc set featurecla = 'Admin-0 country' where ne_id = 1159320903; +update ne_10m_admin_0_countries_tlc set featurecla = 'Admin-0 dependency' where ne_id = 1159320903; -- Fix Saint Helena wikidata id for this build. Remove once NE is updated diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index d83a8b6ac..a96807822 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -8906,7 +8906,7 @@ def tags_set_ne_min_max_zoom(ctx): min_zoom = ceil(min_zoom) props['min_zoom'] = min_zoom - elif props.get('kind') == 'country': + elif props.get('kind') in ('country','unrecognized'): # countries and regions which don't have a min zoom joined from NE # are probably either vandalism or unrecognised countries. either # way, we probably don't want to see them at zoom, which is lower From 4dfcd7167a02635bc692336933fb2c25f49e6580 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 18 May 2022 17:47:55 +0000 Subject: [PATCH 41/56] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- vectordatasource/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index a96807822..dbefd4a0a 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -8906,7 +8906,7 @@ def tags_set_ne_min_max_zoom(ctx): min_zoom = ceil(min_zoom) props['min_zoom'] = min_zoom - elif props.get('kind') in ('country','unrecognized'): + elif props.get('kind') in ('country', 'unrecognized'): # countries and regions which don't have a min zoom joined from NE # are probably either vandalism or unrecognised countries. either # way, we probably don't want to see them at zoom, which is lower From 18efd2eadfd1988fd17f630416ad0b1c70c27a15 Mon Sep 17 00:00:00 2001 From: junderwood Date: Wed, 18 May 2022 14:59:51 -0700 Subject: [PATCH 42/56] set abkhazia to state --- osm2pgsql.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osm2pgsql.lua b/osm2pgsql.lua index 15aac658f..f01c4ecbf 100644 --- a/osm2pgsql.lua +++ b/osm2pgsql.lua @@ -508,7 +508,7 @@ function osm2pgsql.process_node(object) end -- Show Abkhazia label as region for most countries if object.tags.place and object.tags.wikidata == 'Q23334' then - output_hstore['place'] = 'region' + output_hstore['place'] = 'state' output_hstore['place:RU'] = 'country' end -- Turn off South Ossetia label for most countries From 880c5aeac126b5feeceb9cbfcd1f72281bdc0f85 Mon Sep 17 00:00:00 2001 From: junderwood Date: Thu, 19 May 2022 13:05:48 -0700 Subject: [PATCH 43/56] fixes --- data/apply-ne_country_label_recasting.sql | 3 +++ osm2pgsql.lua | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/data/apply-ne_country_label_recasting.sql b/data/apply-ne_country_label_recasting.sql index 217b1b06e..a6ac0efcb 100644 --- a/data/apply-ne_country_label_recasting.sql +++ b/data/apply-ne_country_label_recasting.sql @@ -30,6 +30,9 @@ update ne_10m_admin_0_countries_tlc set featurecla = 'Admin-0 country' where ne_ -- Somaliland update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159321259; +-- Svalbard +update ne_10m_admin_0_countries_tlc set featurecla = 'Admin-0 dependency' where ne_id = 1159321119 + -- Turkish Republic of Northern Cyprus update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159320531; diff --git a/osm2pgsql.lua b/osm2pgsql.lua index bac7b224d..172a4106c 100644 --- a/osm2pgsql.lua +++ b/osm2pgsql.lua @@ -556,6 +556,16 @@ function osm2pgsql.process_node(object) if object.tags.place == 'territory' and object.tags['ISO3166-1'] == 'HM' then output_hstore['place'] = 'country' end +-- Hong Kong + if object.tags.place == 'state' and object.tags['ISO3166-1'] == 'HK' then + output_hstore['place'] = 'country' + output_hstore['place:CN'] = 'state' + end +-- Hong Kong + if object.tags.place == 'state' and object.tags['ISO3166-1'] == 'MO' then + output_hstore['place'] = 'country' + output_hstore['place:CN'] = 'state' + end -- New Caledonia if object.tags.place == 'archipelago' and object.tags['ISO3166-1'] == 'NC' then output_hstore['place'] = 'country' From 892768f0f9fba78c4e868b5e038ed64463787055 Mon Sep 17 00:00:00 2001 From: junderwood Date: Thu, 19 May 2022 16:36:38 -0700 Subject: [PATCH 44/56] typo --- data/apply-ne_country_label_recasting.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/apply-ne_country_label_recasting.sql b/data/apply-ne_country_label_recasting.sql index a6ac0efcb..c63844695 100644 --- a/data/apply-ne_country_label_recasting.sql +++ b/data/apply-ne_country_label_recasting.sql @@ -31,7 +31,7 @@ update ne_10m_admin_0_countries_tlc set featurecla = 'Admin-0 country' where ne_ update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159321259; -- Svalbard -update ne_10m_admin_0_countries_tlc set featurecla = 'Admin-0 dependency' where ne_id = 1159321119 +update ne_10m_admin_0_countries_tlc set featurecla = 'Admin-0 dependency' where ne_id = 1159321119; -- Turkish Republic of Northern Cyprus update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159320531; From 85348157689ffb9dd6f2df2facbb929537392de4 Mon Sep 17 00:00:00 2001 From: junderwood Date: Thu, 19 May 2022 18:00:39 -0700 Subject: [PATCH 45/56] change to dependency --- data/apply-ne_country_label_recasting.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/apply-ne_country_label_recasting.sql b/data/apply-ne_country_label_recasting.sql index c63844695..6ecb6823d 100644 --- a/data/apply-ne_country_label_recasting.sql +++ b/data/apply-ne_country_label_recasting.sql @@ -25,7 +25,7 @@ update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159320709; -- Gaza Strip -update ne_10m_admin_0_countries_tlc set featurecla = 'Admin-0 country' where ne_id = 1159320901; +update ne_10m_admin_0_countries_tlc set featurecla = 'Admin-0 dependency' where ne_id = 1159320901; -- Somaliland update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159321259; From 0896166f86057cc28085b55c34411965f75edd30 Mon Sep 17 00:00:00 2001 From: junderwood Date: Thu, 19 May 2022 19:47:00 -0700 Subject: [PATCH 46/56] Add aland --- osm2pgsql.lua | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/osm2pgsql.lua b/osm2pgsql.lua index 172a4106c..f2b900363 100644 --- a/osm2pgsql.lua +++ b/osm2pgsql.lua @@ -528,6 +528,10 @@ function osm2pgsql.process_node(object) end -- Recast various dependencies as countries +-- Aland + if object.tags.place == 'state' and object.tags['ISO3166-1'] == 'AX' then + output_hstore['place'] = 'country' + end -- American Samoa if object.tags.place == 'state' and object.tags['ISO3166-1'] == 'AS' then output_hstore['place'] = 'country' From f83baa841b6b51311e5af7340a87d533250e3c99 Mon Sep 17 00:00:00 2001 From: junderwood Date: Fri, 20 May 2022 09:02:58 -0700 Subject: [PATCH 47/56] typo --- osm2pgsql.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osm2pgsql.lua b/osm2pgsql.lua index f2b900363..4578aeba3 100644 --- a/osm2pgsql.lua +++ b/osm2pgsql.lua @@ -529,7 +529,7 @@ function osm2pgsql.process_node(object) -- Recast various dependencies as countries -- Aland - if object.tags.place == 'state' and object.tags['ISO3166-1'] == 'AX' then + if object.tags.place == 'county' and object.tags['ISO3166-1'] == 'AX' then output_hstore['place'] = 'country' end -- American Samoa From 8e4ac66a41bb529f0056e6c3666708053aabd0c9 Mon Sep 17 00:00:00 2001 From: junderwood Date: Fri, 20 May 2022 09:32:19 -0700 Subject: [PATCH 48/56] typo --- osm2pgsql.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osm2pgsql.lua b/osm2pgsql.lua index 4578aeba3..1aa164554 100644 --- a/osm2pgsql.lua +++ b/osm2pgsql.lua @@ -565,7 +565,7 @@ function osm2pgsql.process_node(object) output_hstore['place'] = 'country' output_hstore['place:CN'] = 'state' end --- Hong Kong +-- Macau if object.tags.place == 'state' and object.tags['ISO3166-1'] == 'MO' then output_hstore['place'] = 'country' output_hstore['place:CN'] = 'state' From da2fc8fd3a5e3967567deeb9646720dffcd69201 Mon Sep 17 00:00:00 2001 From: junderwood Date: Fri, 20 May 2022 11:41:04 -0700 Subject: [PATCH 49/56] additional countries and country name changes --- osm2pgsql.lua | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/osm2pgsql.lua b/osm2pgsql.lua index 1aa164554..e9c632434 100644 --- a/osm2pgsql.lua +++ b/osm2pgsql.lua @@ -536,6 +536,10 @@ function osm2pgsql.process_node(object) if object.tags.place == 'state' and object.tags['ISO3166-1'] == 'AS' then output_hstore['place'] = 'country' end +-- Bouvet Island + if object.tags.place == 'island' and object.tags['ISO3166-1'] == 'BV' then + output_hstore['place'] = 'country' + end -- Christmas Island if object.tags.place == 'territory' and object.tags['ISO3166-1'] == 'CX' then output_hstore['place'] = 'country' @@ -606,6 +610,10 @@ function osm2pgsql.process_node(object) if object.tags.place == 'region' and object.tags['ISO3166-1'] == 'SJ' then output_hstore['place'] = 'country' end +-- United States Minor Outlying Islands + if object.tags.place == 'region' and object.tags['ISO3166-1'] == 'UM' then + output_hstore['place'] = 'country' + end -- United States Virgin Islands if object.tags.place == 'state' and object.tags['ISO3166-1'] == 'VI' then output_hstore['place'] = 'country' @@ -637,6 +645,19 @@ function osm2pgsql.process_node(object) output_hstore['place'] = 'country' end +-- Update names of some countries for better rendering. We will update the data and deprecate this eventually. +-- Rename Falkland Islands + if object.tags.place == 'country' and object.tags['ISO3166-1'] == 'FK' then + output_hstore['name:en'] = 'Falkland Islands (Islas Malvinas)' + end +-- Rename Saint Martin + if object.tags.place == 'country' and object.tags['ISO3166-1'] == 'SX' then + output_hstore['name:en'] = 'Sint Maarten' + end +-- Rename Sint Maarten + if object.tags.place == 'country' and object.tags['ISO3166-1'] == 'MF' then + output_hstore['name:en'] = 'Saint Martin' + end output.tags = output_hstore From 521f6950c293145fb31de732c939b3a71e703ead Mon Sep 17 00:00:00 2001 From: junderwood Date: Fri, 20 May 2022 13:34:36 -0700 Subject: [PATCH 50/56] corrected labels and place type for saint martin --- osm2pgsql.lua | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/osm2pgsql.lua b/osm2pgsql.lua index e9c632434..dd25fde85 100644 --- a/osm2pgsql.lua +++ b/osm2pgsql.lua @@ -651,12 +651,12 @@ function osm2pgsql.process_node(object) output_hstore['name:en'] = 'Falkland Islands (Islas Malvinas)' end -- Rename Saint Martin - if object.tags.place == 'country' and object.tags['ISO3166-1'] == 'SX' then - output_hstore['name:en'] = 'Sint Maarten' + if object.tags.place == 'state' and object.tags['ISO3166-1'] == 'MF' then + output_hstore['name:en'] = 'Saint Martin' end -- Rename Sint Maarten - if object.tags.place == 'country' and object.tags['ISO3166-1'] == 'MF' then - output_hstore['name:en'] = 'Saint Martin' + if object.tags.place == 'country' and object.tags['ISO3166-1'] == 'SX' then + output_hstore['name:en'] = 'Sint Maarten' end output.tags = output_hstore From aee116ec367f47a102e510c413f7fd67fc9f7196 Mon Sep 17 00:00:00 2001 From: junderwood Date: Fri, 20 May 2022 18:49:05 -0700 Subject: [PATCH 51/56] added max zoom for unrecognized and missing entries in country.yaml --- spreadsheets/min_zoom/country.yaml | 37 ++++++++++++++++++++++++++++-- vectordatasource/transform.py | 1 + 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/spreadsheets/min_zoom/country.yaml b/spreadsheets/min_zoom/country.yaml index bb4ff67ea..70c110987 100644 --- a/spreadsheets/min_zoom/country.yaml +++ b/spreadsheets/min_zoom/country.yaml @@ -279,6 +279,10 @@ FM: FO: min_zoom: 4.0 max_zoom: 9.0 +# France +FR: + min_zoom: 1.7 + max_zoom: 6.7 # Gabon GA: min_zoom: 3.0 @@ -295,6 +299,10 @@ GD: GE: min_zoom: 4.0 max_zoom: 9.0 +# French Guiana +GF: + min_zoom: 5.0 + max_zoom: 10.0 # Guernsey GG: min_zoom: 5.0 @@ -319,6 +327,10 @@ GM: GN: min_zoom: 3.0 max_zoom: 8.0 +# Guadeloupe +GP: + min_zoom: 5.0 + max_zoom: 10.0 # Eq. Guinea GQ: min_zoom: 4.0 @@ -567,6 +579,10 @@ MO: MP: min_zoom: 5.0 max_zoom: 10.0 +# Martinique +MQ: + min_zoom: 5.0 + max_zoom: 10.0 # Mauritania MR: min_zoom: 3.0 @@ -631,6 +647,10 @@ NI: NL: min_zoom: 5.0 max_zoom: 10.0 +# Norway +NO: + min_zoom: 3.0 + max_zoom: 7.0 # Nepal NP: min_zoom: 3.0 @@ -711,6 +731,10 @@ PY: QA: min_zoom: 4.0 max_zoom: 9.0 +# Réunion +RE: + min_zoom: 3.0 + max_zoom: 8.0 # Romania RO: min_zoom: 3.0 @@ -817,8 +841,8 @@ TD: max_zoom: 8.0 # Fr. S. Antarctic Lands TF: - min_zoom: 4.0 - max_zoom: 9.0 + min_zoom: 4.5 + max_zoom: 9.5 # Togo TG: min_zoom: 5.0 @@ -831,6 +855,11 @@ TH: TJ: min_zoom: 4.0 max_zoom: 9.0 +# Tokelau +TK: + min_zoom: 5.0 + max_zoom: 10.0 + # Timor-Leste TL: min_zoom: 4.0 @@ -935,6 +964,10 @@ XK: YE: min_zoom: 3.0 max_zoom: 8.0 +# Mayotte +YT: + min_zoom: 5.0 + max_zoom: 10.0 # South Africa ZA: min_zoom: 1.7 diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index dbefd4a0a..307de5aa8 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -8914,6 +8914,7 @@ def tags_set_ne_min_max_zoom(ctx): # information. props['kind'] = 'unrecognized' props['min_zoom'] = max(8, props['min_zoom']) + props['max_zoom'] = max(10, props['max_zoom']) elif props.get('kind') == 'region': props['min_zoom'] = max(8, props['min_zoom']) From 82ca29e81670a163ad3fb4dc00fd2e761b3b825b Mon Sep 17 00:00:00 2001 From: junderwood Date: Fri, 20 May 2022 22:00:06 -0700 Subject: [PATCH 52/56] fix issue --- vectordatasource/transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index 307de5aa8..8caa7be9f 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -8898,6 +8898,7 @@ def tags_set_ne_min_max_zoom(ctx): for _, props, _ in layer['features']: min_zoom = props.pop('__ne_min_zoom', None) + max_zoom = props.pop('__ne_max_zoom', None) if min_zoom is not None: # don't overstuff features into tiles when they are in the # long tail of won't display, but make their min_zoom @@ -8914,12 +8915,11 @@ def tags_set_ne_min_max_zoom(ctx): # information. props['kind'] = 'unrecognized' props['min_zoom'] = max(8, props['min_zoom']) - props['max_zoom'] = max(10, props['max_zoom']) + props['max_zoom'] = min(10, props['max_zoom']) elif props.get('kind') == 'region': props['min_zoom'] = max(8, props['min_zoom']) - max_zoom = props.pop('__ne_max_zoom', None) if max_zoom is not None: props['max_zoom'] = max_zoom From 8644fbe2388cae765d4c52eb828b699406ab5a1b Mon Sep 17 00:00:00 2001 From: junderwood Date: Fri, 20 May 2022 22:40:09 -0700 Subject: [PATCH 53/56] remove max zoom change for now --- vectordatasource/transform.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vectordatasource/transform.py b/vectordatasource/transform.py index 8caa7be9f..dbefd4a0a 100644 --- a/vectordatasource/transform.py +++ b/vectordatasource/transform.py @@ -8898,7 +8898,6 @@ def tags_set_ne_min_max_zoom(ctx): for _, props, _ in layer['features']: min_zoom = props.pop('__ne_min_zoom', None) - max_zoom = props.pop('__ne_max_zoom', None) if min_zoom is not None: # don't overstuff features into tiles when they are in the # long tail of won't display, but make their min_zoom @@ -8915,11 +8914,11 @@ def tags_set_ne_min_max_zoom(ctx): # information. props['kind'] = 'unrecognized' props['min_zoom'] = max(8, props['min_zoom']) - props['max_zoom'] = min(10, props['max_zoom']) elif props.get('kind') == 'region': props['min_zoom'] = max(8, props['min_zoom']) + max_zoom = props.pop('__ne_max_zoom', None) if max_zoom is not None: props['max_zoom'] = max_zoom From 5506e3a5c59d44cb61f8312d3ce6578ad3c56270 Mon Sep 17 00:00:00 2001 From: junderwood Date: Sat, 21 May 2022 00:59:02 -0700 Subject: [PATCH 54/56] minor formatting improvement --- spreadsheets/min_zoom/country.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spreadsheets/min_zoom/country.yaml b/spreadsheets/min_zoom/country.yaml index 70c110987..e478e9ff4 100644 --- a/spreadsheets/min_zoom/country.yaml +++ b/spreadsheets/min_zoom/country.yaml @@ -330,7 +330,7 @@ GN: # Guadeloupe GP: min_zoom: 5.0 - max_zoom: 10.0 + max_zoom: 10.0 # Eq. Guinea GQ: min_zoom: 4.0 @@ -649,7 +649,7 @@ NL: max_zoom: 10.0 # Norway NO: - min_zoom: 3.0 + min_zoom: 3.0 max_zoom: 7.0 # Nepal NP: @@ -858,7 +858,7 @@ TJ: # Tokelau TK: min_zoom: 5.0 - max_zoom: 10.0 + max_zoom: 10.0 # Timor-Leste TL: From 5f1603b77b9343096dc598ffb355a5d77ed982ea Mon Sep 17 00:00:00 2001 From: junderwood Date: Mon, 23 May 2022 09:45:22 -0700 Subject: [PATCH 55/56] added more countries --- spreadsheets/min_zoom/country.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/spreadsheets/min_zoom/country.yaml b/spreadsheets/min_zoom/country.yaml index e478e9ff4..01052acf2 100644 --- a/spreadsheets/min_zoom/country.yaml +++ b/spreadsheets/min_zoom/country.yaml @@ -127,6 +127,10 @@ BS: BT: min_zoom: 4.0 max_zoom: 9.0 +# Bouvet Island +BV: + min_zoom: 4.5 + max_zoom: 9.5 # Botswana BW: min_zoom: 4.0 @@ -649,8 +653,8 @@ NL: max_zoom: 10.0 # Norway NO: - min_zoom: 3.0 - max_zoom: 7.0 + min_zoom: 3.0 + max_zoom: 7.0 # Nepal NP: min_zoom: 3.0 @@ -783,6 +787,10 @@ SH: SI: min_zoom: 5.0 max_zoom: 10.0 +# Svalbard and Jan Mayen +SJ: + min_zoom: 3.0 + max_zoom: 8.0 # Slovakia SK: min_zoom: 4.0 From a24dd9bcf4cc9d2e38b72dde67c95b6f143d1d99 Mon Sep 17 00:00:00 2001 From: junderwood Date: Mon, 23 May 2022 10:42:37 -0700 Subject: [PATCH 56/56] formatting --- spreadsheets/min_zoom/country.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spreadsheets/min_zoom/country.yaml b/spreadsheets/min_zoom/country.yaml index 01052acf2..380497cf7 100644 --- a/spreadsheets/min_zoom/country.yaml +++ b/spreadsheets/min_zoom/country.yaml @@ -790,7 +790,7 @@ SI: # Svalbard and Jan Mayen SJ: min_zoom: 3.0 - max_zoom: 8.0 + max_zoom: 8.0 # Slovakia SK: min_zoom: 4.0