more rigorous countries labels per ISO and point-of-view (#2069)

This adds sql to alter the tlc table to edit the featurecla column to mark certain geounits as country or unrecognized. OSM labels that match with unrecognized countries or are a non match will return as kind=unrecognized at min zoom 8.

Countries updated as unrecognized

Akrotiri Sovereign Base Area
Ashmore and Cartier Islands
Clipperton Island
Coral Sea Islands
Dhekelia Cantonment
Somaliland
Turkish Republic of Northern Cyprus
Palestine (see Gaza Strip and West Bank instead)
Countries updated to Admin-0 dependency

Gaza Strip
Svalbard
West Bank
Newly rendering countries from data, lua, and sql improvements

Aland
American Samoa
Bermuda
Bouvet Island
Christmas Island
Cocos (Keeling) Islands
French Polynesia
French Southern and Antarctic Lands
Guam
Heard Island and McDonald Islands
Hong Kong
Macau
New Caledonia
Norfolk Island
Northern Mariana Islands
Puerto Rico
Réunion
Saint Helena, Ascension and Tristan da Cunha
Saint Barthélemy
Saint Martin (France)
Saint Pierre and Miquelon
Sāmoa
São Tomé and Príncipe
Svalbard
United States Minor Outlying Islands

data/apply-ne_country_label_recasting.sql

@@ -0,0 +1,44 @@
+-- Recasts a few rows in the ne_10m_admin_0_countries_iso and ne_10m_admin_0_countries_tlc
+-- tables to either show or hide certain labels.
+-- The TLC table yields around 10 country and dependency features over ISO (including Kosovo and Taiwan),
+-- which Tilezen sometimes has opinions about.
+
+
+-- sets featurcla to equal fclass_tlc values. We can then further modify the values while keeping the fclass intact.
+-- featurecla column has a shorter varchar length and needs altering first.
+alter table ne_10m_admin_0_countries_tlc alter column featurecla type varchar;
+update ne_10m_admin_0_countries_tlc set featurecla = fclass_tlc;
+
+-- Akrotiri Sovereign Base Area
+update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159320741;
+
+-- Ashmore and Cartier Islands
+update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159320353;
+
+-- Clipperton Island
+update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159320635;
+
+-- Coral Sea Islands
+update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159320359;
+
+-- Dhekelia Cantonment
+update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159320709;
+
+-- Gaza Strip
+update ne_10m_admin_0_countries_tlc set featurecla = 'Admin-0 dependency' where ne_id = 1159320901;
+
+-- Somaliland
+update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159321259;
+
+-- Svalbard
+update ne_10m_admin_0_countries_tlc set featurecla = 'Admin-0 dependency' where ne_id = 1159321119;
+
+-- Turkish Republic of Northern Cyprus
+update ne_10m_admin_0_countries_tlc set featurecla = 'unrecognized' where ne_id = 1159320531;
+
+-- West Bank
+update ne_10m_admin_0_countries_tlc set featurecla = 'Admin-0 dependency' where ne_id = 1159320903;
+
+
+-- Fix Saint Helena wikidata id for this build. Remove once NE is updated
+update ne_10m_admin_0_countries_iso set wikidataid = 'Q192184' where ne_id = 1159320733;

data/functions.sql

@@ -1017,7 +1017,8 @@ BEGIN
         SELECT
             min_label, max_label INTO min_zoom, max_zoom
         FROM ne_10m_admin_0_countries_tlc t
-        WHERE t.wikidataid = wikidata_id;
+        WHERE t.wikidataid = wikidata_id
+        AND featurecla IN ('Admin-0 country', 'Admin-0 dependency');
       END IF;
   ELSE
     -- try states and provinces if it's not a country
@@ -1071,7 +1072,8 @@ END IF;
         SELECT
             t.fclass_iso, t.fclass_tlc, t.label_x, t.label_y INTO fclass_iso_var, fclass_tlc_var, label_x_var, label_y_var
         FROM ne_10m_admin_0_countries_tlc t
-        WHERE t.wikidataid = wikidata_id;
+        WHERE t.wikidataid = wikidata_id
+        AND featurecla IN ('Admin-0 country', 'Admin-0 dependency');
     END IF;
 
     IF NOT FOUND THEN

data/perform-sql-updates.sh

@@ -47,6 +47,11 @@ psql $PSQLOPTS  $@ -f apply-planet_disputed_area_features_name_suppression.sql
 echo -e "\nRecasting unwanted NE disputed lines to unrecognized"
 psql $PSQLOPTS  $@ -f apply-ne_disputed_border_suppression.sql
 
+# Recasts a few rows in the ne_10m_admin_0_countries_iso and ne_10m_admin_0_countries_tlc
+# tables to either show or hide certain labels.
+echo -e "\nRecasting NE country labels"
+psql $PSQLOPTS  $@ -f apply-ne_country_label_recasting.sql
+
 # Australia suburbs are treated more like cities than typical US style suburbs so we recast them to place=town
 echo -e "\nRecasting Australia suburbs"
 psql $PSQLOPTS  $@ -f apply-planet_australia_suburb_recast.sql

integration-test/1826-unrecognised-country-labels.py

@@ -37,8 +37,8 @@ def test_country_no_ne(self):
 
         self.assert_has_feature(
             z, x, y, 'places', {
-                'kind': 'country',
-                'min_zoom': 6,
+                'kind': 'unrecognized',
+                'min_zoom': 8,
             })
 
     def test_region_no_ne(self):

osm2pgsql.lua

@@ -508,7 +508,7 @@ function osm2pgsql.process_node(object)
     end
 -- Show Abkhazia label as region for most countries
     if object.tags.place and object.tags.wikidata == 'Q23334' then
-        output_hstore['place'] = 'region'
+        output_hstore['place'] = 'state'
         output_hstore['place:RU'] = 'country'
     end
 -- Turn off South Ossetia label for most countries
@@ -528,10 +528,18 @@ function osm2pgsql.process_node(object)
     end
 
 -- Recast various dependencies as countries
+-- Aland
+    if object.tags.place == 'county' and object.tags['ISO3166-1'] == 'AX' then
+        output_hstore['place'] = 'country'
+    end
 -- American Samoa
     if object.tags.place == 'state' and object.tags['ISO3166-1'] == 'AS' then
         output_hstore['place'] = 'country'
     end
+-- Bouvet Island
+    if object.tags.place == 'island' and object.tags['ISO3166-1'] == 'BV' then
+        output_hstore['place'] = 'country'
+    end
 -- Christmas Island
     if object.tags.place == 'territory' and object.tags['ISO3166-1'] == 'CX' then
         output_hstore['place'] = 'country'
@@ -556,8 +564,18 @@ function osm2pgsql.process_node(object)
     if object.tags.place == 'territory' and object.tags['ISO3166-1'] == 'HM' then
         output_hstore['place'] = 'country'
     end
+-- Hong Kong
+    if object.tags.place == 'state' and object.tags['ISO3166-1'] == 'HK' then
+        output_hstore['place'] = 'country'
+        output_hstore['place:CN'] = 'state'
+    end
+-- Macau
+    if object.tags.place == 'state' and object.tags['ISO3166-1'] == 'MO' then
+        output_hstore['place'] = 'country'
+        output_hstore['place:CN'] = 'state'
+    end
 -- New Caledonia
-    if object.tags.place == 'archipelago' and object.tags['ISO3166-1'] == 'NF' then
+    if object.tags.place == 'archipelago' and object.tags['ISO3166-1'] == 'NC' then
         output_hstore['place'] = 'country'
     end
 -- Norfolk Island
@@ -592,6 +610,10 @@ function osm2pgsql.process_node(object)
     if object.tags.place == 'region' and object.tags['ISO3166-1'] == 'SJ' then
         output_hstore['place'] = 'country'
     end
+-- United States Minor Outlying Islands
+    if object.tags.place == 'region' and object.tags['ISO3166-1'] == 'UM' then
+        output_hstore['place'] = 'country'
+    end
 -- United States Virgin Islands
     if object.tags.place == 'state' and object.tags['ISO3166-1'] == 'VI' then
         output_hstore['place'] = 'country'
@@ -623,6 +645,19 @@ function osm2pgsql.process_node(object)
         output_hstore['place'] = 'country'
     end
 
+-- Update names of some countries for better rendering. We will update the data and deprecate this eventually.
+-- Rename Falkland Islands
+    if object.tags.place == 'country' and object.tags['ISO3166-1'] == 'FK' then
+        output_hstore['name:en'] = 'Falkland Islands (Islas Malvinas)'
+    end
+-- Rename Saint Martin
+    if object.tags.place == 'state' and object.tags['ISO3166-1'] == 'MF' then
+        output_hstore['name:en'] = 'Saint Martin'
+    end
+-- Rename Sint Maarten
+    if object.tags.place == 'country' and object.tags['ISO3166-1'] == 'SX' then
+        output_hstore['name:en'] = 'Sint Maarten'
+    end
 
     output.tags = output_hstore
 

spreadsheets/min_zoom/country.yaml

@@ -127,6 +127,10 @@ BS:
 BT:
   min_zoom:  4.0
   max_zoom:  9.0
+# Bouvet Island
+BV:
+  min_zoom:  4.5
+  max_zoom:  9.5
 # Botswana
 BW:
   min_zoom:  4.0
@@ -279,6 +283,10 @@ FM:
 FO:
   min_zoom:  4.0
   max_zoom:  9.0
+# France
+FR:
+  min_zoom:  1.7
+  max_zoom:  6.7
 # Gabon
 GA:
   min_zoom:  3.0
@@ -295,6 +303,10 @@ GD:
 GE:
   min_zoom:  4.0
   max_zoom:  9.0
+# French Guiana
+GF:
+  min_zoom:  5.0
+  max_zoom: 10.0
 # Guernsey
 GG:
   min_zoom:  5.0
@@ -319,6 +331,10 @@ GM:
 GN:
   min_zoom:  3.0
   max_zoom:  8.0
+# Guadeloupe
+GP:
+  min_zoom:  5.0
+  max_zoom: 10.0
 # Eq. Guinea
 GQ:
   min_zoom:  4.0
@@ -567,6 +583,10 @@ MO:
 MP:
   min_zoom:  5.0
   max_zoom: 10.0
+# Martinique
+MQ:
+  min_zoom:  5.0
+  max_zoom: 10.0
 # Mauritania
 MR:
   min_zoom:  3.0
@@ -631,6 +651,10 @@ NI:
 NL:
   min_zoom:  5.0
   max_zoom: 10.0
+# Norway
+NO:
+  min_zoom:  3.0
+  max_zoom:  7.0
 # Nepal
 NP:
   min_zoom:  3.0
@@ -711,6 +735,10 @@ PY:
 QA:
   min_zoom:  4.0
   max_zoom:  9.0
+# Réunion
+RE:
+  min_zoom:  3.0
+  max_zoom:  8.0
 # Romania
 RO:
   min_zoom:  3.0
@@ -759,6 +787,10 @@ SH:
 SI:
   min_zoom:  5.0
   max_zoom: 10.0
+# Svalbard and Jan Mayen
+SJ:
+  min_zoom:  3.0
+  max_zoom:  8.0
 # Slovakia
 SK:
   min_zoom:  4.0
@@ -817,8 +849,8 @@ TD:
   max_zoom:  8.0
 # Fr. S. Antarctic Lands
 TF:
-  min_zoom:  4.0
-  max_zoom:  9.0
+  min_zoom:  4.5
+  max_zoom:  9.5
 # Togo
 TG:
   min_zoom:  5.0
@@ -831,6 +863,11 @@ TH:
 TJ:
   min_zoom:  4.0
   max_zoom:  9.0
+# Tokelau
+TK:
+  min_zoom:  5.0
+  max_zoom: 10.0
+
 # Timor-Leste
 TL:
   min_zoom:  4.0
@@ -935,6 +972,10 @@ XK:
 YE:
   min_zoom:  3.0
   max_zoom:  8.0
+# Mayotte
+YT:
+  min_zoom:  5.0
+  max_zoom:  10.0
 # South Africa
 ZA:
   min_zoom:  1.7

vectordatasource/transform.py

@@ -8906,13 +8906,14 @@ def tags_set_ne_min_max_zoom(ctx):
                 min_zoom = ceil(min_zoom)
             props['min_zoom'] = min_zoom
 
-        elif props.get('kind') == 'country':
+        elif props.get('kind') in ('country', 'unrecognized'):
             # countries and regions which don't have a min zoom joined from NE
             # are probably either vandalism or unrecognised countries. either
             # way, we probably don't want to see them at zoom, which is lower
             # than most of the curated NE min zooms. see issue #1826 for more
             # information.
-            props['min_zoom'] = max(6, props['min_zoom'])
+            props['kind'] = 'unrecognized'
+            props['min_zoom'] = max(8, props['min_zoom'])
 
         elif props.get('kind') == 'region':
             props['min_zoom'] = max(8, props['min_zoom'])

yaml/places.yaml

@@ -147,7 +147,11 @@ filters:
     #
     # note: also, we knock this down to 6 during post-processing for any country
     # which doesn't match an NE curated one.
-    min_zoom: 1
+    min_zoom:
+      case:
+        - when: { place: country }
+          then: 1
+        - else: 8
     output:
       <<: [*output_properties, *ne_zooms]
       kind: {col: place}