From adc08d1b499aacc9256b7bfa5ffda6cc50e402e1 Mon Sep 17 00:00:00 2001 From: deepgabani8 Date: Mon, 2 Oct 2023 12:41:11 +0000 Subject: [PATCH 1/2] Not initializing ee for every worker if it has already been initialized before. --- weather_mv/loader_pipeline/ee.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/weather_mv/loader_pipeline/ee.py b/weather_mv/loader_pipeline/ee.py index 094b3a17..e117c4de 100644 --- a/weather_mv/loader_pipeline/ee.py +++ b/weather_mv/loader_pipeline/ee.py @@ -157,7 +157,12 @@ def setup(self): def check_setup(self): """Ensures that setup has been called.""" if not self._has_setup: - self.setup() + try: + # This throws an exception if ee is not initialized. + ee.data.getAlgorithms() + self._has_setup = True + except ee.EEException: + self.setup() def process(self, *args, **kwargs): """Checks that setup has been called then call the process implementation.""" From b1cec58df09d08428a2ba0765df02319d5c9dc7e Mon Sep 17 00:00:00 2001 From: deepgabani8 Date: Mon, 2 Oct 2023 16:26:17 +0000 Subject: [PATCH 2/2] Bumped version number --- weather_mv/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weather_mv/setup.py b/weather_mv/setup.py index b46121a5..4bdb4a0b 100644 --- a/weather_mv/setup.py +++ b/weather_mv/setup.py @@ -65,7 +65,7 @@ packages=find_packages(), author='Anthromets', author_email='anthromets-ecmwf@google.com', - version='0.2.18', + version='0.2.19', url='https://weather-tools.readthedocs.io/en/latest/weather_mv/', description='A tool to load weather data into BigQuery.', install_requires=beam_gcp_requirements + base_requirements,