Improves editing delta layers

XYZHubConnector/xyz_qgis/layer/edit_buffer.py

@@ -214,6 +214,7 @@ def get_layer_id(self):
         return self.layer_id
 
     def get_sync_feat(self):
+        is_livemap = self.get_conn_info().is_livemap()
         added_ids, removed_ids = self.get_ids()
 
         m = self.get_xyz_id_(added_ids)
@@ -228,7 +229,7 @@ def get_sync_feat(self):
         vlayer = get_layer(self.layer_id)
         it = vlayer.getFeatures(added_ids)
         lst_added_feat, added_ids = get_feat_upload_from_iter(
-            it, vlayer, lst_fid=added_ids, lst_xyz_id=added_xyz_ids
+            it, vlayer, lst_fid=added_ids, lst_xyz_id=added_xyz_ids, is_livemap=is_livemap
         )
         lst_removed_ids = parser.make_lst_removed_ids(removed_xyz_ids)
 

XYZHubConnector/xyz_qgis/layer/layer.py

@@ -103,15 +103,26 @@ def load_from_qnode(cls, qnode):
         obj._update_group_name(qnode)
 
         # obj._save_meta_node(qnode)
-        for i in qnode.findLayers():
-            vlayer = i.layer()
+
+        for q in qnode.findLayers():
+            vlayer = q.layer()
             if not vlayer.isValid():
                 continue
-            geom_str = QgsWkbTypes.displayString(vlayer.wkbType())
-            obj.map_vlayer.setdefault(geom_str, list()).append(vlayer)
-            obj.map_fields.setdefault(geom_str, list()).append(vlayer.dataProvider().fields())
-            obj.update_constraint_trigger(geom_str, len(obj.map_vlayer[geom_str]) - 1)
-            # obj._save_meta_vlayer(vlayer)
+            geom_str, idx = obj.geom_str_idx_from_vlayer(vlayer)
+
+            lst_vlayer = obj.map_vlayer.setdefault(geom_str, list())
+            lst_fields = obj.map_fields.setdefault(geom_str, list())
+            while len(lst_vlayer) < idx + 1:
+                lst_vlayer.append(None)
+                lst_fields.append(parser.new_fields_gpkg())
+            lst_vlayer[idx] = vlayer
+            lst_fields[idx] = vlayer.dataProvider().fields()
+
+            obj.update_constraint_trigger(geom_str, idx)
+            vlayer_geom_str = QgsWkbTypes.displayString(vlayer.wkbType())
+            if vlayer_geom_str and not vlayer_geom_str.endswith("Z"):
+                obj.update_z_geom(geom_str, idx, vlayer)
+            vlayer.reload()
         return obj
 
     def _save_params_to_node(self, qnode):
@@ -197,6 +208,8 @@ def cb(*a):
         return cb
 
     def _connect_cb_vlayer(self, vlayer, geom_str, idx):
+        if vlayer is None:
+            return
         cb_delete_vlayer = self.callbacks.setdefault(
             vlayer.id(), self._make_cb_args(self._cb_delete_vlayer, vlayer, geom_str, idx)
         )
@@ -295,10 +308,26 @@ def _layer_name(self, geom_str, idx):
 
     def _db_layer_name(self, geom_str, idx):
         """
-        returns name of the table corresponds to vlayer in sqlite db
+        returns name of the table corresponds to vlayer in gpkg/sqlite db
         """
         return "{geom}_{idx}".format(geom=geom_str, idx=idx)
 
+    def _parse_db_layer_name(self, db_layer_name):
+        """
+        returns geom_str, idx from table name in gpkg/sqlite db
+        """
+        geom_str, idx = db_layer_name.split("_", 1)
+        return geom_str, int(idx)
+
+    def geom_str_idx_from_vlayer(self, vlayer):
+        """
+        returns geom_str, idx from vlayer
+        """
+        uri = vlayer.source()
+        key = "|layername="
+        db_layer_name = uri[uri.find(key) + len(key) :]
+        return self._parse_db_layer_name(db_layer_name)
+
     def _layer_fname(self):
         """
         returns file name of the sqlite db corresponds to xyz layer
@@ -465,9 +494,12 @@ def _init_ext_layer(self, geom_str, idx, crs):
 
         # fname = make_unique_full_path(ext=ext)
         fname = make_fixed_full_path(self._layer_fname(), ext=ext)
-
+        if geom_str:
+            geomz = geom_str if geom_str.endswith("Z") else "{}Z".format(geom_str)
+        else:
+            geomz = "NoGeometry"
         vlayer = QgsVectorLayer(
-            "{geom}?crs={crs}&index=yes".format(geom=geom_str, crs=crs), layer_name, "memory"
+            "{geom}?crs={crs}&index=yes".format(geom=geomz, crs=crs), layer_name, "memory"
         )  # this should be done in main thread
 
         # QgsVectorFileWriter.writeAsVectorFormat(vlayer, fname, "UTF-8", vlayer.sourceCrs(),
@@ -531,6 +563,22 @@ def _update_constraint_trigger(self, fname, layer_name):
         conn.commit()
         conn.close()
 
+    def update_z_geom(self, geom_str, idx, vlayer):
+        db_layer_name = self._db_layer_name(geom_str, idx)
+        fname = make_fixed_full_path(self._layer_fname(), ext=self.ext)
+        sql = """
+        UPDATE "gpkg_geometry_columns" SET "z"=1 WHERE "table_name" = "{layer_name}";
+        """.format(
+            layer_name=db_layer_name
+        )
+        conn = sqlite3.connect(fname)
+        cur = conn.cursor()
+        cur.execute(sql)
+        conn.commit()
+        conn.close()
+        # set to the same data source to apply changes
+        vlayer.setDataSource(vlayer.source(), vlayer.sourceName(), "ogr")
+
 
 """ Available vector format for QgsVectorFileWriter
 [i.driverName for i in QgsVectorFileWriter.ogrDriverList()]

XYZHubConnector/xyz_qgis/layer/layer_utils.py

@@ -46,7 +46,9 @@ def get_feat_upload_from_iter_args(feat_iter, vlayer):
     return make_qt_args(*a)
 
 
-def get_feat_upload_from_iter(feat_iter, vlayer, lst_fid: list = None, lst_xyz_id: list = None):
+def get_feat_upload_from_iter(
+    feat_iter, vlayer, lst_fid: list = None, lst_xyz_id: list = None, is_livemap=False
+):
     """get feature as geojson from iter. Also return lst_fid ordering
     optinal input: lst_fid order and lst_xyz_id mapping
     ensure same order as lst_fid
@@ -70,7 +72,7 @@ def get_feat_upload_from_iter(feat_iter, vlayer, lst_fid: list = None, lst_xyz_i
     if transformer.isValid() and not transformer.isShortCircuited():
         lst_feat = filter(None, (parser.transform_geom(ft, transformer) for ft in lst_feat))
 
-    added_feat = parser.feature_to_xyz_json(lst_feat, is_new=False)
+    added_feat = parser.feature_to_xyz_json(lst_feat, is_new=False, is_livemap=is_livemap)
     for ft, xyz_id in zip(added_feat, lst_xyz_id):
         if ft is None or xyz_id is None:
             continue

XYZHubConnector/xyz_qgis/layer/parser.py

@@ -151,10 +151,10 @@ def check_non_expression_fields(fields):
     return all(fields.fieldOrigin(i) != fields.OriginExpression for i, f in enumerate(fields))
 
 
-def feature_to_xyz_json(features, is_new=False, ignore_null=True):
+def feature_to_xyz_json(features, is_new=False, ignore_null=True, is_livemap=False):
     def _xyz_props(props, ignore_keys=tuple()):
-        # for all key start with @ (internal): str to dict (disabled)
-        # k = "@ns:com:here:xyz"
+        # convert from qgs case insensitive fields back to json properties
+        # convert from json string to json object
         new_props = dict()
         for t in props.keys():
             if ignore_null and props[t] is None:
@@ -163,11 +163,6 @@ def _xyz_props(props, ignore_keys=tuple()):
 
             if k in ignore_keys:
                 continue
-            # drop @ fields for upload
-            if k.startswith("@ns:com:here:xyz"):
-                continue
-            if k.startswith("@") and k != "@ns:com:here:mom:delta":
-                continue
             new_props[k] = props[t]
 
             # always handle json string in props
@@ -178,12 +173,23 @@ def _xyz_props(props, ignore_keys=tuple()):
                     new_props[k] = json.loads(v)
                 except json.JSONDecodeError:
                     pass
-            # handle editing of delta layer
-            if k == "@ns:com:here:mom:delta":
-                if "reviewState" in new_props[k]:
-                    new_props[k]["reviewState"] = ""  # UNPUBLISHED
         return new_props
 
+    def _livemap_props(props, xyz_id=None):
+        # handle editing of delta layer
+        changeState = "UPDATED" if "@ns:com:here:mom:delta" in props else "CREATED"
+        delta = {"reviewState": "UNPUBLISHED", "changeState": changeState, "taskGridId": ""}
+        if xyz_id:
+            delta.update({"originId": xyz_id})
+        return {"@ns:com:here:mom:delta": delta}
+
+    def _clean_props(props):
+        # drop @ fields for upload
+        ignored_special_keys = [k for k in props.keys() if k.startswith("@")]
+        for k in ignored_special_keys:
+            props.pop(k, None)
+        return props
+
     def _single_feature(feat):
         # existing feature json
         if feat is None:
@@ -207,8 +213,9 @@ def _single_feature(feat):
         ]
         # print({k.name(): fields.fieldOrigin(i) for i, k in enumerate(fields)})
         props = _xyz_props(props, ignore_keys=expression_field_names)
-
-        obj["properties"] = props
+        livemap_props = _livemap_props(props, xyz_id=obj.get(XYZ_ID)) if is_livemap else dict()
+        props = _clean_props(props)
+        obj["properties"] = dict(props, **livemap_props)
 
         geom = feat.geometry()
         geom_ = json.loads(geom.asJson())

XYZHubConnector/xyz_qgis/layer/render.py

@@ -127,6 +127,7 @@ def add_feature_render(vlayer, feat, new_fields):
     ok, out_feat = pr.addFeatures(feat)
     vlayer.updateExtents()  # will hide default progress bar
     # post_render(vlayer) # disable in order to keep default progress bar running
+    vlayer.reload()
     return ok, out_feat
 
 

XYZHubConnector/xyz_qgis/models/connection.py

@@ -22,6 +22,8 @@ def parse_copyright(v):
 
 class SpaceConnectionInfo(object):
     EXCLUDE_PROJECT_KEYS = ["here_client_secret"]
+    LIVEMAP = "LIVEMAP"
+    LIVEMAP_CID = "3fN7oveDupmTGsr5mUM5"
 
     def __init__(self, conn_info=None):
         if conn_info is None:
@@ -74,3 +76,9 @@ def to_project_dict(self):
         for ex in self.EXCLUDE_PROJECT_KEYS:
             d.pop(ex, "")
         return d
+
+    def is_livemap(self):
+        packages = self.get_("packages", list())
+        check_pkg = any(p for p in packages if self.LIVEMAP in p)
+        check_cid = self.get_("cid") == self.LIVEMAP_CID
+        return check_pkg or check_cid

test/test_parser.py

@@ -41,6 +41,7 @@ def test_parse_xyzjson(self):
             self.subtest_parse_xyzjson(folder, fname)
 
     def subtest_parse_xyzjson(self, folder, fname):
+        feat = list()
         with self.subTest(folder=folder, fname=fname):
             resource = TestFolder(folder)
 
@@ -52,6 +53,7 @@ def subtest_parse_xyzjson(self, folder, fname):
 
             self._assert_parsed_fields(obj_feat, feat, fields)
             self._assert_parsed_geom(obj_feat, feat, fields)
+        return feat
 
     def _assert_parsed_fields_unorder(self, obj_feat, feat, fields):
         # self._log_debug(fields.names())
@@ -433,12 +435,15 @@ def test_parse_xyzjson_map_large(self):
 
     ######## Parse QgsFeature -> json
     def test_parse_qgsfeature(self):
-        self.subtest_parse_qgsfeature("geojson-small", "airport-qgis.geojson")  # no xyz_id
+        # self.subtest_parse_qgsfeature("geojson-small", "airport-qgis.geojson")  # no xyz_id
+        self.subtest_parse_qgsfeature("xyzjson-small", "airport-xyz.geojson")
+        self.subtest_parse_qgsfeature_2way("xyzjson-small", "airport-xyz.geojson")
+        self.subtest_parse_qgsfeature_livemap("xyzjson-small", "livemap-xyz.geojson")
 
     def subtest_parse_qgsfeature(self, folder, fname):
         # qgs layer load geojson -> qgs feature
-        # parse feature to xyz geojson
-        # compare geojson and xyzgeojson
+        # parse feature to geojson
+        # compare geojson and geojson
         with self.subTest(folder=folder, fname=fname):
 
             resource = TestFolder(folder)
@@ -452,8 +457,99 @@ def subtest_parse_qgsfeature(self, folder, fname):
             )  # remove QGS_XYZ_ID if exist
             self._log_debug(feat)
 
-            self.assertListEqual(obj["features"], feat)
-            self.assertEqual(len(obj["features"]), len(feat))
+            self.maxDiff = None
+            # no need to convert 0.0 to 0
+            expected = obj
+            for ft in expected["features"]:
+                ft.pop("id", None)
+                ft["properties"].pop("@ns:com:here:xyz", None)
+            for ft in feat:
+                ft["properties"].pop("id", None)  # cleanup unexpected "id" field in input data
+            self.assertListEqual(expected["features"], feat)
+            self.assertEqual(len(expected["features"]), len(feat))
+
+    def subtest_parse_qgsfeature_2way(self, folder, fname):
+        # parse xyz geojson to qgs feature
+        # parse feature to xyz geojson
+        # compare geojson and xyz geojson
+        with self.subTest(folder=folder, fname=fname, mode="2way", target="QgsFeature"):
+            qgs_feat = self.subtest_parse_xyzjson(folder, fname)
+
+        with self.subTest(folder=folder, fname=fname, mode="2way", target="XYZ Geojson"):
+
+            resource = TestFolder(folder)
+            txt = resource.load(fname)
+            obj = json.loads(txt)
+            expected = obj
+
+            feat = parser.feature_to_xyz_json(qgs_feat)
+            self._log_debug(feat)
+
+            self.maxDiff = None
+            # no need to convert 0.0 to 0
+            for ft in expected["features"]:
+                ft["properties"].pop("@ns:com:here:xyz", None)
+            self.assertListEqual(expected["features"], feat)
+            self.assertEqual(len(expected["features"]), len(feat))
+
+            feat = parser.feature_to_xyz_json(qgs_feat, is_new=True)
+            self._log_debug(feat)
+
+            for ft in expected["features"]:
+                ft.pop("id", None)
+            self.assertListEqual(expected["features"], feat)
+            self.assertEqual(len(expected["features"]), len(feat))
+
+    def subtest_parse_qgsfeature_livemap(self, folder, fname):
+        # test parse livemap qgsfeature
+        with self.subTest(folder=folder, fname=fname, mode="livemap", target="QgsFeature"):
+            qgs_feat = self.subtest_parse_xyzjson(folder, fname)
+
+        with self.subTest(folder=folder, fname=fname, mode="livemap", target="XYZ Geojson"):
+            resource = TestFolder(folder)
+            txt = resource.load(fname)
+            obj = json.loads(txt)
+
+            feat = parser.feature_to_xyz_json(qgs_feat, is_livemap=True)
+
+            self.maxDiff = None
+            expected = obj
+            for ft in expected["features"]:
+                ft.pop("momType", None)
+                props = ft["properties"]
+
+                changeState = "UPDATED" if "@ns:com:here:mom:delta" in props else "CREATED"
+                delta = {
+                    "reviewState": "UNPUBLISHED",
+                    "changeState": changeState,
+                    "taskGridId": "",
+                }
+                if ft.get("id"):
+                    delta.update({"originId": ft.get("id")})
+
+                ignored_sepcial_keys = [k for k in props.keys() if k.startswith("@")]
+                ignored_keys = [k for k, v in props.items() if v is None]
+                for k in ignored_sepcial_keys + ignored_keys:
+                    props.pop(k, None)
+
+                props.update({"@ns:com:here:mom:delta": delta})
+
+            lst_coords_ref = [
+                ft.pop("geometry", dict()).get("coordinates", list())
+                for ft in expected["features"]
+            ]
+            lst_coords = [ft.pop("geometry", dict()).get("coordinates", list()) for ft in feat]
+
+            self.assertListEqual(expected["features"], feat)
+            self.assertEqual(len(expected["features"]), len(feat))
+
+            # self.assertEqual(flatten(lst_coords_ref), flatten(lst_coords))
+            for coords_ref, coords in zip(lst_coords_ref, lst_coords):
+                self.assertLess(
+                    np.max(np.abs(np.array(coords_ref) - np.array(coords))),
+                    1e-13,
+                    "parsed geometry error > 1e-13",
+                )
 
     def test_parse_qgsfeature_large(self):
         pass