Update webservices.py (#314)

Two issues are solved:
- If values for a large area are prompted, the query url gets very long and is not accepted by HHNK polderpeil server. Is now queried in chunks
- An error in the database of HHNK resulted in duplicate peil definitions. Only use the unambigious entries and leave a note in the logs.

nlmod/read/webservices.py

@@ -150,22 +150,53 @@ def arcrest(
         else:
             gdf = gpd.GeoDataFrame.from_features(features, crs=sr)
             if table is not None:
-                url_query = f"{url}/{table.pop('id')}/query"
-                pgbids = ",".join([str(v) for v in gdf["OBJECTID"].values])
-                params["where"] = f"PEILGEBIEDVIGERENDID IN ({pgbids})"
                 params["f"] = "json"
-                data = _get_data(url_query, params, timeout=timeout)
+                url_query = f"{url}/{table.pop('id')}/query"
+
+                # loop over chunks of 100 pgbids. Long where clauses can cause
+                # the request to fail. 1300 pgbids fails but 130 works
+                chunk_size = 100
+                ids_chunks = [
+                    gdf["OBJECTID"].values[i : i + chunk_size]
+                    for i in range(0, len(gdf), chunk_size)
+                ]
+                data = {}
+                features = []
+
+                for ids_chunk in ids_chunks:
+                    pgbids = ",".join([str(v) for v in ids_chunk])
+                    where = f"PEILGEBIEDVIGERENDID IN ({pgbids})"
+                    params["where"] = where
+                    _data = _get_data(url_query, params, timeout=timeout, **kwargs)
+
+                    data.update(_data)
+                    features.extend(_data["features"])
+
+                assert "exceededTransferLimit" not in data, "exceededTransferLimit"
+                data["features"] = features
+
                 df = pd.DataFrame(
                     [feature["attributes"] for feature in data["features"]]
                 )
+
                 # add peilen to gdf
                 for col, convert_dic in table.items():
                     df[col].replace(convert_dic, inplace=True)
                     df.set_index(col, inplace=True)
+
                     for oid in gdf["OBJECTID"]:
                         insert_s = df.loc[
                             df["PEILGEBIEDVIGERENDID"] == oid, "WATERHOOGTE"
                         ]
+                        if insert_s.index.duplicated().any():
+                            # Error in the database. Reported to Doeke HHNK 20230123
+                            # Continue with unambiguous values
+                            dup = set(insert_s.index[insert_s.index.duplicated()])
+                            logger.warning(
+                                f"Duplicate {dup} values for PEILGEBIEDVIGERENDID {oid} while prompting {url}"
+                            )
+                            insert_s = insert_s[~insert_s.index.duplicated(keep=False)]
+
                         gdf.loc[
                             gdf["OBJECTID"] == oid, insert_s.index
                         ] = insert_s.values