more rigorous countries labels per ISO and point-of-view

data/apply-ne_country_label_recasting.sql

@@ -0,0 +1,41 @@
+-- Recasts a few rows in the ne_10m_admin_0_countries_iso and ne_10m_admin_0_countries_tlc
+-- tables to either show or hide certain labels
+
+
+-- Akrotiri Sovereign Base Area
+update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320741;
+
+-- Aland
+update ne_10m_admin_0_countries_iso set fclass_iso = 'unrecognized' where ne_id = 1159320621;
+update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320621;
+
+-- Antarctica
+update ne_10m_admin_0_countries_iso set fclass_iso = 'unrecognized' where ne_id = 1159320335;
+update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320335;
+
+-- Ashmore and Cartier Islands
+update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320353;
+
+-- Clipperton Island
+update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320635;
+
+-- Coral Sea Islands
+update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320359;
+
+-- Dhekelia Cantonment
+update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320709;
+
+-- Gaza Strip
+update ne_10m_admin_0_countries_tlc set fclass_tlc = 'Admin-0 country' where ne_id = 1159320901;
+
+-- Palestine
+update ne_10m_admin_0_countries_iso set fclass_iso = 'unrecognized' where ne_id = 1159320899;
+
+-- Somaliland
+update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159321259;
+
+-- Turkish Republic of Northern Cyprus
+update ne_10m_admin_0_countries_tlc set fclass_tlc = 'unrecognized' where ne_id = 1159320531;
+
+-- West Bank
+update ne_10m_admin_0_countries_tlc set fclass_tlc = 'Admin-0 country' where ne_id = 1159320903;

data/assets.yaml

@@ -1,5 +1,5 @@
 bucket: nextzen-tile-assets
-datestamp: 20220426
+datestamp: 20220505
 
 shapefiles:
 
@@ -55,98 +55,107 @@ shapefiles:
     url: http://s3.amazonaws.com/tilezen-assets/curated/admin_areas_20180409.zip
 
   - name: ne_110m_lakes
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/physical/ne_110m_lakes.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/110m/physical/ne_110m_lakes.zip
 
   - name: ne_50m_lakes
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/physical/ne_50m_lakes.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/50m/physical/ne_50m_lakes.zip
 
   - name: ne_10m_lakes
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_lakes.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/10m/physical/ne_10m_lakes.zip
 
   - name: ne_110m_ocean
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/physical/ne_110m_ocean.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/110m/physical/ne_110m_ocean.zip
 
   - name: ne_50m_ocean
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/physical/ne_50m_ocean.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/50m/physical/ne_50m_ocean.zip
 
   - name: ne_10m_ocean
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_ocean.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/10m/physical/ne_10m_ocean.zip
 
   - name: ne_50m_playas
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/physical/ne_50m_playas.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/50m/physical/ne_50m_playas.zip
 
   - name: ne_10m_playas
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_playas.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/10m/physical/ne_10m_playas.zip
 
   - name: ne_50m_urban_areas
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_urban_areas.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/50m/cultural/ne_50m_urban_areas.zip
 
   - name: ne_10m_urban_areas
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_urban_areas.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_urban_areas.zip
 
   - name: ne_110m_land
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/physical/ne_110m_land.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/110m/physical/ne_110m_land.zip
 
   - name: ne_50m_land
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/physical/ne_50m_land.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/50m/physical/ne_50m_land.zip
 
   - name: ne_10m_land
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_land.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/10m/physical/ne_10m_land.zip
     tile: true
 
   - name: ne_10m_populated_places
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_populated_places.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_populated_places.zip
 
   - name: ne_110m_admin_0_boundary_lines_land
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/cultural/ne_110m_admin_0_boundary_lines_land.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/110m/cultural/ne_110m_admin_0_boundary_lines_land.zip
 
   - name: ne_50m_admin_0_boundary_lines_land
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_0_boundary_lines_land.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/50m/cultural/ne_50m_admin_0_boundary_lines_land.zip
 
   - name: ne_10m_admin_0_boundary_lines_land
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_boundary_lines_land.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_boundary_lines_land.zip
 
   - name: ne_10m_admin_0_boundary_lines_map_units
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_boundary_lines_map_units.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_boundary_lines_map_units.zip
 
   - name: ne_50m_admin_1_states_provinces_lines
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_1_states_provinces_lines.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/50m/cultural/ne_50m_admin_1_states_provinces_lines.zip
 
   - name: ne_10m_admin_1_states_provinces_lines
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_1_states_provinces_lines.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_1_states_provinces_lines.zip
 
   - name: ne_50m_admin_0_boundary_lines_disputed_areas
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_0_boundary_lines_disputed_areas.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/50m/cultural/ne_50m_admin_0_boundary_lines_disputed_areas.zip
 
   - name: ne_50m_admin_0_boundary_lines_maritime_indicator_chn
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_0_boundary_lines_maritime_indicator_chn.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/50m/cultural/ne_50m_admin_0_boundary_lines_maritime_indicator_chn.zip
 
   - name: ne_10m_admin_0_boundary_lines_disputed_areas
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_boundary_lines_disputed_areas.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_boundary_lines_disputed_areas.zip
 
   - name: ne_10m_admin_0_boundary_lines_maritime_indicator_chn
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_boundary_lines_maritime_indicator_chn.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_boundary_lines_maritime_indicator_chn.zip
 
   - name: ne_10m_roads
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_roads.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_roads.zip
 
   - name: ne_110m_coastline
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/physical/ne_110m_coastline.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/110m/physical/ne_110m_coastline.zip
 
   - name: ne_50m_coastline
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/physical/ne_50m_coastline.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/50m/physical/ne_50m_coastline.zip
 
   - name: ne_10m_coastline
-    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_coastline.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/10m/physical/ne_10m_coastline.zip
 
   - name: ne_10m_admin_0_countries
-    url: https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_countries.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_countries.zip
+    prj: 3857
+
+  - name: ne_10m_admin_0_countries_iso
+    url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_countries_iso.zip
+    prj: 3857
+
+  - name: ne_10m_admin_0_countries_tlc
+    url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_countries_tlc.zip
+    prj: 3857
 
   - name: ne_10m_admin_0_map_units
-    url: https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_map_units.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_0_map_units.zip
 
   - name: ne_10m_admin_1_states_provinces
-    url: https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_1_states_provinces.zip
+    url: https://naciscdn.org/naturalearth/5.1.0/10m/cultural/ne_10m_admin_1_states_provinces.zip
 
 wikidata-queries:
   - name: aerodrome_passenger_count

data/functions.sql

@@ -996,7 +996,7 @@ $$ LANGUAGE plpgsql IMMUTABLE;
 -- returns a JSONB object containing __ne_min_zoom and __ne_max_zoom set to the
 -- label min and max zoom of any matching row from the Natural Earth countries,
 -- map units and states/provinces themes.
-CREATE OR REPLACE FUNCTION tz_get_ne_min_max_zoom(wikidata_id TEXT)
+CREATE OR REPLACE FUNCTION tz_get_ne_min_max_zoom(wikidata_id TEXT, place_tag TEXT)
 RETURNS JSONB AS $$
 DECLARE
   min_zoom REAL;
@@ -1006,28 +1006,31 @@ BEGIN
     RETURN '{}'::jsonb;
   END IF;
 
-  -- first, try the countries table
-  SELECT
-    min_label, max_label INTO min_zoom, max_zoom
-    FROM ne_10m_admin_0_countries c
-    WHERE c.wikidataid = wikidata_id;
+  -- if it's a country, only look it up in the iso and tlc tables
+  IF place_tag='country' OR place_tag='unrecognized' THEN
+      SELECT
+        min_label, max_label INTO min_zoom, max_zoom
+        FROM ne_10m_admin_0_countries_iso i
+        WHERE i.wikidataid = wikidata_id;
 
-  -- if that fails, try map_units (which contains some sub-country but super-
-  -- state level stuff such as England, Scotland and Wales).
-  IF NOT FOUND THEN
-    SELECT
-      min_label, max_label INTO min_zoom, max_zoom
-      FROM ne_10m_admin_0_map_units mu
-      WHERE mu.wikidataid = wikidata_id;
+      IF NOT FOUND THEN
+        SELECT
+            min_label, max_label INTO min_zoom, max_zoom
+        FROM ne_10m_admin_0_countries_tlc t
+        WHERE t.wikidataid = wikidata_id
+        AND fclass_tlc IN ('Admin-0 country' or 'Admin-0 dependency');
+      END IF;
+
+      IF NOT FOUND THEN
+        RETURN '{}'::jsonb;
+      END IF;
   END IF;
 
-  -- try states and provinces
-  IF NOT FOUND THEN
-    SELECT
+  -- try states and provinces if it's not a country
+  SELECT
       min_label, max_label INTO min_zoom, max_zoom
       FROM ne_10m_admin_1_states_provinces sp
       WHERE sp.wikidataid = wikidata_id;
-  END IF;
 
   -- finally, try localities
   -- There is no concept of max_zoom for ne_10m_populated_places
@@ -1051,6 +1054,66 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql STABLE;
 
+CREATE OR REPLACE FUNCTION tz_get_fclass_and_label_position(wikidata_id TEXT, place_tag TEXT)
+RETURNS JSONB AS $$
+DECLARE
+fclass_iso TEXT;
+fclass_tlc TEXT;
+label_x REAL;
+label_y REAL;
+BEGIN
+  IF wikidata_id IS NULL THEN
+    RETURN '{}'::jsonb;
+END IF;
+
+  -- if it's a country, only look it up in the iso and tlc tables
+  IF place_tag='country' OR place_tag='unrecognized' THEN
+    SELECT
+        fclass_iso, fclass_tlc, label_x, label_y INTO fclass_iso, fclass_tlc, label_x, label_y
+    FROM ne_10m_admin_0_countries_iso i
+    WHERE i.wikidataid = wikidata_id;
+
+    IF NOT FOUND THEN
+        SELECT
+            fclass_iso, fclass_tlc, label_x, label_y INTO fclass_iso, fclass_tlc, label_x, label_y
+        FROM ne_10m_admin_0_countries_tlc t
+        WHERE t.wikidataid = wikidata_id
+        AND fclass_tlc IN ('Admin-0 country' or 'Admin-0 dependency');
+    END IF;
+
+    IF NOT FOUND THEN
+        RETURN '{}'::jsonb;
+    END IF;
+  END IF;
+
+  -- There is no label_x and label_y for the non-countries
+  SELECT
+    fclass_iso, fclass_tlc, NULL, NULL INTO fclass_iso, fclass_tlc, label_x, label_y
+  FROM ne_10m_admin_1_states_provinces sp
+  WHERE sp.wikidataid = wikidata_id;
+
+  -- finally, try localities
+  IF NOT FOUND THEN
+  SELECT
+    fclass_iso, fclass_tlc, NULL, NULL INTO fclass_iso, fclass_tlc, label_x, label_y
+  FROM ne_10m_populated_places pp
+  WHERE pp.wikidataid = wikidata_id;END IF;
+
+  -- return an empty JSONB rather than null, so that it can be safely
+  -- concatenated with whatever other JSONB rather than needing a check for
+  -- null.
+  IF NOT FOUND THEN
+    RETURN '{}'::jsonb;
+  END IF;
+  RETURN jsonb_build_object(
+        '__ne_fclass_iso', fclass_iso,
+        '__ne_fclass_tlc', fclass_tlc,
+        '__ne_label_x', label_x,
+        '__ne_label_y', label_y
+    );
+END;
+$$ LANGUAGE plpgsql STABLE;
+
 
 -- returns a JSONB object containing __ne_pop_min and __ne_pop_max
 CREATE OR REPLACE FUNCTION tz_get_ne_pop_min_max(wikidata_id TEXT)

data/perform-sql-updates.sh

@@ -47,6 +47,11 @@ psql $PSQLOPTS  $@ -f apply-planet_disputed_area_features_name_suppression.sql
 echo -e "\nRecasting unwanted NE disputed lines to unrecognized"
 psql $PSQLOPTS  $@ -f apply-ne_disputed_border_suppression.sql
 
+# Recasts a few rows in the ne_10m_admin_0_countries_iso and ne_10m_admin_0_countries_tlc
+# tables to either show or hide certain labels.
+echo -e "\nRecasting NE country labels"
+psql $PSQLOPTS  $@ -f apply-ne_country_label_recasting.sql
+
 # Australia suburbs are treated more like cities than typical US style suburbs so we recast them to place=town
 echo -e "\nRecasting Australia suburbs"
 psql $PSQLOPTS  $@ -f apply-planet_australia_suburb_recast.sql

integration-test/1826-unrecognised-country-labels.py

@@ -37,7 +37,7 @@ def test_country_no_ne(self):
 
         self.assert_has_feature(
             z, x, y, 'places', {
-                'kind': 'country',
+                'kind': 'unrecognized',
                 'min_zoom': 6,
             })
 

queries/ne.jinja2

@@ -1,5 +1,5 @@
 {% set ne_languages = ['ar', 'bn', 'de', 'el', 'en', 'es', 'fa', 'fr', 'he', 'hi', 'hu', 'id', 'it', 'ja', 'ko', 'nl', 'pl', 'pt', 'ru', 'sv', 'tr', 'uk', 'ur', 'vi', 'zh', 'zht'] %}
-{% set ne_viewpoints = ['iso', 'ar', 'bd', 'br', 'cn', 'de', 'eg', 'es', 'fr', 'gb', 'gr', 'id', 'il', 'in', 'it', 'jp', 'ko', 'ma', 'nl', 'np', 'pk', 'pl', 'ps', 'pt', 'ru', 'sa', 'se', 'tr', 'tw', 'ua', 'us', 'vn'] %}
+{% set ne_viewpoints = ['iso', 'tlc', 'ar', 'bd', 'br', 'cn', 'de', 'eg', 'es', 'fr', 'gb', 'gr', 'id', 'il', 'in', 'it', 'jp', 'ko', 'ma', 'nl', 'np', 'pk', 'pl', 'ps', 'pt', 'ru', 'sa', 'se', 'tr', 'tw', 'ua', 'us', 'vn'] %}
 
 {% macro ne_common_properties() %}
 

queries/planet_osm_point.jinja2

@@ -27,9 +27,11 @@ SELECT
     THEN jsonb_build_object(
       'min_zoom', mz_places_min_zoom
     ) ||
-      tz_get_ne_min_max_zoom(tags->'wikidata')
+      tz_get_ne_min_max_zoom(tags->'wikidata', tags->'place')
       ||
       tz_get_ne_pop_min_max(tags->'wikidata')
+      ||
+      tz_get_fclass_and_label_position(tags->'wikidata', tags->'place')
   END AS __places_properties__,
 
   CASE WHEN mz_poi_min_zoom IS NOT NULL

vectordatasource/transform.py

@@ -8912,7 +8912,8 @@ def tags_set_ne_min_max_zoom(ctx):
             # way, we probably don't want to see them at zoom, which is lower
             # than most of the curated NE min zooms. see issue #1826 for more
             # information.
-            props['min_zoom'] = max(6, props['min_zoom'])
+            props['kind'] = 'unrecognized'
+            props['min_zoom'] = max(8, props['min_zoom'])
 
         elif props.get('kind') == 'region':
             props['min_zoom'] = max(8, props['min_zoom'])
@@ -9295,7 +9296,7 @@ def _list_of_countries(value):
         # should have an ISO 3166-1 alpha-2 code, so should be 2 ASCII
         # latin characters.
         candidate = candidate.strip().lower()
-        if candidate == 'iso' or match('[a-z][a-z]', candidate):
+        if candidate == 'iso' or candidate == 'tlc' or match('[a-z][a-z]', candidate):
             countries.append(candidate)
 
     return countries

yaml/boundaries.yaml

@@ -29,6 +29,7 @@ global:
 
   - &ne_localized_kind_properties
     'kind:iso': {col: fclass_iso }
+    'kind:tlc': {col: fclass_tlc }
     'kind:ar': {col: fclass_ar }
     'kind:bd': {col: fclass_bd }
     'kind:br': {col: fclass_br }

yaml/places.yaml

@@ -36,6 +36,7 @@ global:
     wikidata_id: {col: wikidata}
     disputed_by: {col: disputed_by }
     'place:ISO': { col: 'place:ISO' }
+    'place:TLC': { col: 'place:TLC' }
     'place:AR': { col: 'place:AR' }
     'place:BD': { col: 'place:BD' }
     'place:BR': { col: 'place:BR' }
@@ -83,6 +84,7 @@ global:
 
   - &alternate_fclass
     fclass_iso: { col: fclass_iso }
+    fclass_tlc: { col: fclass_tlc }
     fclass_ar: { col: fclass_ar }
     fclass_bd: { col: fclass_bd }
     fclass_br: { col: fclass_br }