Workaround: For weather-mv, fix installation step for non-regrid pipelines.

.github/workflows/conda-beam-image.yml

@@ -0,0 +1,55 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+name: Miniconda Beam Image
+
+on:
+  # Triggers the workflow on push or pull request events but only for the main branch
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+env:
+  PROJECT: ai-for-weather
+  REPO: miniconda3-beam
+  TAG: "0.0.2"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.7" "3.8", "3.9"]
+        beam-sdk: ["2.38.0", "2.39.0", "2.40.0"]
+
+    # Add "id-token" with the intended permissions.
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+
+    steps:
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v0'
+        with:
+          workload_identity_provider: 'projects/123456789/locations/global/workloadIdentityPools/my-pool/providers/my-provider'
+          service_account: 'my-service-account@my-project.iam.gserviceaccount.com'
+
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v0'
+
+      - name: 'Use gcloud CLI'
+        run: 'gcloud info'

weather_mv/Dockerfile → Dockerfile

@@ -16,13 +16,18 @@
 ARG py_version=3.8
 FROM apache/beam_python${py_version}_sdk:2.40.0 as beam_sdk
 FROM continuumio/miniconda3:4.12.0
-ARG py_version
 
 # Update miniconda
 RUN conda update conda -y
 
-# Install desired python version
-RUN conda install python=${py_version} -y
+# Create conda env using environment.yml
+COPY environment.yml /tmp/environment.yml
+RUN conda env create -f /tmp/environment.yml
+
+# Activate the conda env and update the PATH
+ARG CONDA_ENV_NAME=weather-tools
+RUN echo "source activate ${CONDA_ENV_NAME}" >> ~/.bashrc
+ENV PATH /opt/conda/envs/${CONDA_ENV_NAME}/bin:$PATH
 
 # Install SDK.
 RUN pip install --no-cache-dir apache-beam[gcp]==2.40.0

environment.yml

@@ -20,6 +20,7 @@ dependencies:
   - rioxarray
   - pyproj
   - gdal
+  - scipy
   - pygrib
   - flake8
   - pytest
@@ -34,4 +35,5 @@ dependencies:
     - google-cloud-bigquery
     - urllib3==1.26.5
     - earthengine-api>=0.1.263
-    - google-cloud-storage==2.2.1
+    - google-cloud-storage==2.2.1
+    - pytype==2021.11.29

setup.py

@@ -104,7 +104,7 @@
              'weather_mv/weather-mv', 'weather_sp/weather-sp'],
     tests_require=test_requirements,
     extras_require={
-        'dev': ['tox', 'sphinx>=2.1', 'myst-parser'] + all_test_requirements,
+        'dev': ['tox', 'sphinx>=2.1', 'myst-parser'],
         'test': all_test_requirements,
         'regrid': ['metview']
     },

weather_mv/setup.py

@@ -27,23 +27,23 @@
 from setuptools import setup, find_packages, Command
 
 base_requirements = [
-    "apache-beam[gcp]",
-    "dataclasses",
-    "numpy",
-    "pandas",
-    "xarray",
-    "google-cloud-storage==2.2.1",
-    "cfgrib",
-    "netcdf4",
-    "geojson",
-    "more-itertools",
-    "simplejson",
-    "rioxarray",
-    "metview",
-    "rasterio",
-    "earthengine-api>=0.1.263",
-    "pyproj",  # requires separate binary installation!
-    "gdal",  # requires separate binary installation!
+    # "apache-beam[gcp]",
+    # "dataclasses",
+    # "numpy",
+    # "pandas",
+    # "xarray",
+    # "google-cloud-storage==2.2.1",
+    # "cfgrib",
+    # "netcdf4",
+    # "geojson",
+    # "more-itertools",
+    # "simplejson",
+    # "rioxarray",
+    # # "metview",
+    # "rasterio",
+    # "earthengine-api>=0.1.263",
+    # # "pyproj",  # requires separate binary installation!
+    # # "gdal",  # requires separate binary installation!
 ]
 
 
@@ -88,10 +88,9 @@ class build(_build):  # pylint: disable=invalid-name
 """Install the ecCodes and MetView packages from ECMWF."""
 CUSTOM_COMMANDS = [
     cmd.split() for cmd in [
-        'apt-get update',
-        'apt-get --assume-yes install libeccodes-dev',
-        'conda install gdal -c conda-forge -y',
-        'conda install metview-batch -c conda-forge -y',
+        # 'conda install eccodes -c conda-forge -y',
+        # 'conda install gdal -c conda-forge -y',
+        # 'conda install metview-batch -c conda-forge -y',
     ]
 ]
 
@@ -117,7 +116,7 @@ def RunCustomCommand(self, command_list):
         stdout_data, _ = p.communicate()
         print('Command output: %s' % stdout_data)
         if p.returncode != 0:
-            raise print(
+            raise RuntimeError(
                 'Command %s failed: exit code: %s' % (command_list, p.returncode))
 
     def run(self):
@@ -133,10 +132,10 @@ def run(self):
     version='0.2.3',
     url='https://weather-tools.readthedocs.io/en/latest/weather_mv/',
     description='A tool to load weather data into BigQuery.',
-    install_requires=base_requirements,
-    cmdclass={
-        # Command class instantiated and run during pip install scenarios.
-        'build': build,
-        'CustomCommands': CustomCommands,
-    }
+    # install_requires=base_requirements,
+    # cmdclass={
+    #     # Command class instantiated and run during pip install scenarios.
+    #     'build': build,
+    #     'CustomCommands': CustomCommands,
+    # }
 )