From d6be74a92980879cc5fa4b81234952c6009e12ef Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 6 Jul 2022 18:10:06 -0700 Subject: [PATCH 1/7] Return 0 if conda is not found. --- weather_mv/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weather_mv/setup.py b/weather_mv/setup.py index 4d65cc96..88c242d5 100644 --- a/weather_mv/setup.py +++ b/weather_mv/setup.py @@ -87,7 +87,7 @@ class build(_build): # pylint: disable=invalid-name cmd.split() for cmd in [ 'apt-get update', 'apt-get --assume-yes install libeccodes-dev', - 'conda install metview-batch -c conda-forge -y', + 'conda install metview-batch -c conda-forge -y || true', ] ] From b4b29c55dd2648d24752a09d163c284034645247 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Wed, 6 Jul 2022 23:06:51 -0700 Subject: [PATCH 2/7] Omit raising an error. --- weather_mv/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weather_mv/setup.py b/weather_mv/setup.py index 88c242d5..ff5bbcf9 100644 --- a/weather_mv/setup.py +++ b/weather_mv/setup.py @@ -87,7 +87,7 @@ class build(_build): # pylint: disable=invalid-name cmd.split() for cmd in [ 'apt-get update', 'apt-get --assume-yes install libeccodes-dev', - 'conda install metview-batch -c conda-forge -y || true', + 'conda install metview-batch -c conda-forge -y', ] ] @@ -113,7 +113,7 @@ def RunCustomCommand(self, command_list): stdout_data, _ = p.communicate() print('Command output: %s' % stdout_data) if p.returncode != 0: - raise print( + print( 'Command %s failed: exit code: %s' % (command_list, p.returncode)) def run(self): From 7ed34126b03c60f4e29765959c0a88fc517e3cfa Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Sun, 17 Jul 2022 12:47:28 -0700 Subject: [PATCH 3/7] WIP Creating a GH action to build the beam miniconda image. --- .cloudignore | 0 .github/workflows/conda-beam-image.yml | 55 ++++++++++++++++++++++++++ weather_mv/setup.py | 2 +- 3 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 .cloudignore create mode 100644 .github/workflows/conda-beam-image.yml diff --git a/.cloudignore b/.cloudignore new file mode 100644 index 00000000..e69de29b diff --git a/.github/workflows/conda-beam-image.yml b/.github/workflows/conda-beam-image.yml new file mode 100644 index 00000000..9eb218a2 --- /dev/null +++ b/.github/workflows/conda-beam-image.yml @@ -0,0 +1,55 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: Miniconda Beam Image + +on: + # Triggers the workflow on push or pull request events but only for the main branch + push: + branches: [ main ] + pull_request: + branches: [ main ] + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +env: + PROJECT: ai-for-weather + REPO: miniconda3-beam + TAG: "0.0.2" + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.7" "3.9", "3.9"] + beam-sdk: ["2.38.0", "2.39.0", "2.40.0"] + + # Add "id-token" with the intended permissions. + permissions: + contents: 'read' + id-token: 'write' + + steps: + - id: 'auth' + uses: 'google-github-actions/auth@v0' + with: + workload_identity_provider: 'projects/123456789/locations/global/workloadIdentityPools/my-pool/providers/my-provider' + service_account: 'my-service-account@my-project.iam.gserviceaccount.com' + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v0' + + - name: 'Use gcloud CLI' + run: 'gcloud info' diff --git a/weather_mv/setup.py b/weather_mv/setup.py index 737bb349..0d8d79ce 100644 --- a/weather_mv/setup.py +++ b/weather_mv/setup.py @@ -117,7 +117,7 @@ def RunCustomCommand(self, command_list): stdout_data, _ = p.communicate() print('Command output: %s' % stdout_data) if p.returncode != 0: - print( + raise RuntimeError( 'Command %s failed: exit code: %s' % (command_list, p.returncode)) def run(self): From b1c420434011838c7ab2254c93e262555a687723 Mon Sep 17 00:00:00 2001 From: Alex Merose Date: Sun, 17 Jul 2022 13:09:49 -0700 Subject: [PATCH 4/7] Nit: need to also include python 3.8. --- .github/workflows/conda-beam-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda-beam-image.yml b/.github/workflows/conda-beam-image.yml index 9eb218a2..60bf0540 100644 --- a/.github/workflows/conda-beam-image.yml +++ b/.github/workflows/conda-beam-image.yml @@ -33,7 +33,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7" "3.9", "3.9"] + python-version: ["3.7" "3.8", "3.9"] beam-sdk: ["2.38.0", "2.39.0", "2.40.0"] # Add "id-token" with the intended permissions. From 13b8a096331982a1fddff71f736feed4aa58ca94 Mon Sep 17 00:00:00 2001 From: mahrsee1997 Date: Mon, 25 Jul 2022 22:32:48 +0530 Subject: [PATCH 5/7] updated Dockerfile to create conda env using environment.yml file --- weather_mv/Dockerfile => Dockerfile | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) rename weather_mv/Dockerfile => Dockerfile (78%) diff --git a/weather_mv/Dockerfile b/Dockerfile similarity index 78% rename from weather_mv/Dockerfile rename to Dockerfile index 209d0642..a96713c6 100644 --- a/weather_mv/Dockerfile +++ b/Dockerfile @@ -16,13 +16,18 @@ ARG py_version=3.8 FROM apache/beam_python${py_version}_sdk:2.40.0 as beam_sdk FROM continuumio/miniconda3:4.12.0 -ARG py_version # Update miniconda RUN conda update conda -y -# Install desired python version -RUN conda install python=${py_version} -y +# Create conda env using environment.yml +COPY environment.yml /tmp/environment.yml +RUN conda env create -f /tmp/environment.yml + +# Activate the conda env and update the PATH +ARG CONDA_ENV_NAME=weather-tools +RUN echo "source activate ${CONDA_ENV_NAME}" >> ~/.bashrc +ENV PATH /opt/conda/envs/${CONDA_ENV_NAME}/bin:$PATH # Install SDK. RUN pip install --no-cache-dir apache-beam[gcp]==2.40.0 From 225558b92cce65edfc24d14136fbfe7a544d341a Mon Sep 17 00:00:00 2001 From: mahrsee1997 Date: Mon, 25 Jul 2022 22:33:50 +0530 Subject: [PATCH 6/7] updated environment.yml file & update setup.py for dev installation --- environment.yml | 4 +++- setup.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 944f836a..b69b39c5 100644 --- a/environment.yml +++ b/environment.yml @@ -20,6 +20,7 @@ dependencies: - rioxarray - pyproj - gdal + - scipy - pygrib - flake8 - pytest @@ -34,4 +35,5 @@ dependencies: - google-cloud-bigquery - urllib3==1.26.5 - earthengine-api>=0.1.263 - - google-cloud-storage==2.2.1 \ No newline at end of file + - google-cloud-storage==2.2.1 + - pytype==2021.11.29 \ No newline at end of file diff --git a/setup.py b/setup.py index 094e0935..2a05d939 100644 --- a/setup.py +++ b/setup.py @@ -104,7 +104,7 @@ 'weather_mv/weather-mv', 'weather_sp/weather-sp'], tests_require=test_requirements, extras_require={ - 'dev': ['tox', 'sphinx>=2.1', 'myst-parser'] + all_test_requirements, + 'dev': ['tox', 'sphinx>=2.1', 'myst-parser'], 'test': all_test_requirements, 'regrid': ['metview'] }, From 881212a295eea6efa54c3e03e18b231346ad5b68 Mon Sep 17 00:00:00 2001 From: mahrsee1997 Date: Fri, 5 Aug 2022 23:02:20 +0530 Subject: [PATCH 7/7] temporary fix for running weather-mv dataflow job --- weather_mv/setup.py | 53 ++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/weather_mv/setup.py b/weather_mv/setup.py index 84aa7980..9a9752a4 100644 --- a/weather_mv/setup.py +++ b/weather_mv/setup.py @@ -27,23 +27,23 @@ from setuptools import setup, find_packages, Command base_requirements = [ - "apache-beam[gcp]", - "dataclasses", - "numpy", - "pandas", - "xarray", - "google-cloud-storage==2.2.1", - "cfgrib", - "netcdf4", - "geojson", - "more-itertools", - "simplejson", - "rioxarray", - "metview", - "rasterio", - "earthengine-api>=0.1.263", - "pyproj", # requires separate binary installation! - "gdal", # requires separate binary installation! + # "apache-beam[gcp]", + # "dataclasses", + # "numpy", + # "pandas", + # "xarray", + # "google-cloud-storage==2.2.1", + # "cfgrib", + # "netcdf4", + # "geojson", + # "more-itertools", + # "simplejson", + # "rioxarray", + # # "metview", + # "rasterio", + # "earthengine-api>=0.1.263", + # # "pyproj", # requires separate binary installation! + # # "gdal", # requires separate binary installation! ] @@ -88,10 +88,9 @@ class build(_build): # pylint: disable=invalid-name """Install the ecCodes and MetView packages from ECMWF.""" CUSTOM_COMMANDS = [ cmd.split() for cmd in [ - 'apt-get update', - 'apt-get --assume-yes install libeccodes-dev', - 'conda install gdal -c conda-forge -y', - 'conda install metview-batch -c conda-forge -y', + # 'conda install eccodes -c conda-forge -y', + # 'conda install gdal -c conda-forge -y', + # 'conda install metview-batch -c conda-forge -y', ] ] @@ -133,10 +132,10 @@ def run(self): version='0.2.3', url='https://weather-tools.readthedocs.io/en/latest/weather_mv/', description='A tool to load weather data into BigQuery.', - install_requires=base_requirements, - cmdclass={ - # Command class instantiated and run during pip install scenarios. - 'build': build, - 'CustomCommands': CustomCommands, - } + # install_requires=base_requirements, + # cmdclass={ + # # Command class instantiated and run during pip install scenarios. + # 'build': build, + # 'CustomCommands': CustomCommands, + # } )