bug fix: RuntimeError from NetCDF lib w/ zlib=True on vlens (#10)

Avoid passing zlib=True to the NetCDF library for vlen datatypes. One day the library decided this wasn't okay... 🤷.
5tefan · Jul 6, 2023 · d6df44b · d6df44b
1 parent 3f4c3b8
commit d6df44b
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 5 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,7 +1,9 @@
-# 0.8.16 - 2023-06-14
+# 0.8.17 - 2023-07-05
 
 - New feature: support `copy_from_alt` list of alternative variable names to use if `name` variable
-isn't present.
+  isn't present.
+- Fix bug: avoid passing `zlib=True` on str datatypes to avoid RuntimeError from NetCDF library
+  Unidata/netcdf4-python#1205
 
 # 0.8.15 - 2022-11-30
 

diff --git a/ncagg/aggregator.py b/ncagg/aggregator.py
@@ -384,13 +384,19 @@ def initialize_aggregation_file(config, fullpath):
                 # fill_value is None by default, but if there is a value specified,
                 # explicitly cast it to the same type as the data.
                 fill_value = var_type.type(fill_value)
+            zlib = True
+            if np.issubdtype(var_type, str):
+                # NetCDF started raising RuntimeError when passed compression args on 
+                # vlen datatypes. Detect vlens (str for now) and avoid compression.
+                # Ref: https://github.com/Unidata/netcdf4-python/issues/1205
+                zlib = False
             var_out = nc_out.createVariable(
                 var_name,
                 var_type,
                 var["dimensions"],
                 chunksizes=var["chunksizes"],
-                zlib=True,
-                complevel=7,
+                zlib=zlib,
+                complevel=7 if zlib else None,
                 fill_value=fill_value,
             )
             for k, v in var["attributes"].items():

diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 
 setup(
     name="ncagg",
-    version="0.8.16",
+    version="0.8.17",
     description="Utility for aggregation of NetCDF data.",
     author="Stefan Codrescu",
     author_email="stefan.codrescu@noaa.gov",
@@ -23,6 +23,7 @@
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
         "License :: OSI Approved :: MIT License",
         "Operating System :: OS Independent",
     ],