From d3e0c56325136abf5d828b12d81b0137bccfa3cd Mon Sep 17 00:00:00 2001 From: michaelchin Date: Tue, 14 Nov 2023 17:38:58 +1100 Subject: [PATCH] add nest_asyncio to deal with multiple event loops --- README.md | 6 ++ pyproject.toml | 3 +- src/plate_model_manager/network_aiohttp.py | 8 +- src/plate_model_manager/network_requests.py | 15 +++- src/plate_model_manager/plate_model.py | 16 +++- unittest/test_nest_asyncio.ipynb | 83 +++++++++++++++++++++ 6 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 unittest/test_nest_asyncio.ipynb diff --git a/README.md b/README.md index 63b45d9..a5fc56a 100644 --- a/README.md +++ b/README.md @@ -82,3 +82,9 @@ The examples of using PlateModelManager with GPlately: - https://github.com/GPlates/gplately/blob/master/Notebooks/Examples/working-with-plate-model-manager.py The PlateModelManager can also be used with the GPlates desktop. Use the command line to download the plate model files and open the files with GPlates desktop. This will save the trouble of downloading files from Internet manually. + +### Dependencies + +- aiohttp +- requests +- nest_asyncio diff --git a/pyproject.toml b/pyproject.toml index 2d16c03..3b40943 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "plate-model-manager" -version = "1.1.0" +version = "1.1.1" description = "Plate Model Manager" readme = "README.md" authors = [{ name = "Michael Chin", email = "michael.chin@sydney.edu.au" }] @@ -18,6 +18,7 @@ keywords = ["Tectonics", "Earth"] dependencies = [ "aiohttp", "requests", + "nest_asyncio", ] requires-python = ">=3.8" diff --git a/src/plate_model_manager/network_aiohttp.py b/src/plate_model_manager/network_aiohttp.py index 756fcd0..513e9d8 100644 --- a/src/plate_model_manager/network_aiohttp.py +++ b/src/plate_model_manager/network_aiohttp.py @@ -58,7 +58,7 @@ async def _fetch_file( ): """async "fetch_file" implementation. See the docstring of "fetch_file" """ - if etag: + if isinstance(etag, str) or isinstance(etag, bytes): headers = {"If-None-Match": etag} else: headers = {} @@ -206,7 +206,11 @@ async def f(): new_etags = [] try: new_etags = loop.run_until_complete(f()) - # print(new_etags) + except RuntimeError: + import nest_asyncio + + nest_asyncio.apply() + new_etags = loop.run_until_complete(f()) finally: loop.close() return new_etags diff --git a/src/plate_model_manager/network_requests.py b/src/plate_model_manager/network_requests.py index e4b0bd8..53bded6 100644 --- a/src/plate_model_manager/network_requests.py +++ b/src/plate_model_manager/network_requests.py @@ -39,7 +39,7 @@ def fetch_file( """ - if etag: + if isinstance(etag, str) or isinstance(etag, bytes): headers = {"If-None-Match": etag} else: headers = {} @@ -203,6 +203,19 @@ def fetch_files( auto_unzip=auto_unzip, ) ) + except RuntimeError: + import nest_asyncio + + nest_asyncio.apply() + loop.run_until_complete( + self._async_fetch_files( + run, + urls, + filepaths, + etags=etags, + auto_unzip=auto_unzip, + ) + ) finally: loop.close() diff --git a/src/plate_model_manager/plate_model.py b/src/plate_model_manager/plate_model.py index b36747a..8eb08d9 100644 --- a/src/plate_model_manager/plate_model.py +++ b/src/plate_model_manager/plate_model.py @@ -301,7 +301,13 @@ async def f(): # print(tasks) await asyncio.wait(tasks) - self.loop.run_until_complete(f()) + try: + self.loop.run_until_complete(f()) + except RuntimeError: + import nest_asyncio + + nest_asyncio.apply() + self.loop.run_until_complete(f()) def get_avail_time_dependent_raster_names(self): """return the names of all time dependent rasters which have been configurated in this model.""" @@ -345,7 +351,13 @@ async def f(): # print(tasks) await asyncio.wait(tasks) - self.loop.run_until_complete(f()) + try: + self.loop.run_until_complete(f()) + except RuntimeError: + import nest_asyncio + + nest_asyncio.apply() + self.loop.run_until_complete(f()) else: raise Exception( diff --git a/unittest/test_nest_asyncio.ipynb b/unittest/test_nest_asyncio.ipynb new file mode 100644 index 0000000..0991abc --- /dev/null +++ b/unittest/test_nest_asyncio.ipynb @@ -0,0 +1,83 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "2c3659e1-ede1-4a79-b0bb-5346c5336300", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "downloading https://www.earthbyte.org/webdav/ftp/Data_Collections/Muller_etal_2019_Tectonics/Muller_etal_2019_Agegrids/Muller_etal_2019_Tectonics_v2.0_netCDF/Muller_etal_2019_Tectonics_v2.0_AgeGrid-0.ncdownloading https://www.earthbyte.org/webdav/ftp/Data_Collections/Muller_etal_2019_Tectonics/Muller_etal_2019_Agegrids/Muller_etal_2019_Tectonics_v2.0_netCDF/Muller_etal_2019_Tectonics_v2.0_AgeGrid-1.nc\n", + "\n", + "The local file(s) is/are still good. Will not download again at this moment.\n", + "The local file(s) is/are still good. Will not download again at this moment.\n", + "downloading https://repo.gplates.org/webdav/pmm/muller2019/Coastlines.zip\n", + "downloading https://repo.gplates.org/webdav/pmm/muller2019/Rotations.zip\n", + "downloading https://repo.gplates.org/webdav/pmm/muller2019/ContinentalPolygons.zip\n", + "downloading https://repo.gplates.org/webdav/pmm/muller2019/StaticPolygons.zip\n", + "downloading https://repo.gplates.org/webdav/pmm/muller2019/Topologies.zip\n", + "downloading https://repo.gplates.org/webdav/pmm/muller2019/COBs.zip\n", + "The local file(s) is/are still good. Will not download again at this moment.\n", + "The local file(s) is/are still good. Will not download again at this moment.\n", + "The local file(s) is/are still good. Will not download again at this moment.\n", + "The local file(s) is/are still good. Will not download again at this moment.\n", + "The local file(s) is/are still good. Will not download again at this moment.\n", + "The local file(s) is/are still good. Will not download again at this moment.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/mchin/workspace/plate-model-manager.git/src/plate_model_manager/plate_model.py:360: RuntimeWarning: coroutine 'PlateModel.download_time_dependent_rasters..f' was never awaited\n" + ] + } + ], + "source": [ + "#import nest_asyncio\n", + "#nest_asyncio.apply()\n", + "#you need to uncomment the two lines above if you don't what to see a warning\n", + "\n", + "import sys,os\n", + "sys.path.insert(0, f\"/Users/mchin/workspace/plate-model-manager.git/src\")\n", + "from plate_model_manager import PlateModelManager\n", + "pm_manager = PlateModelManager()\n", + "muller2019_model = pm_manager.get_model(\"Muller2019\", data_dir=\"plate-model-repo\")\n", + "muller2019_model.get_rasters(\"AgeGrids\", [0,1])\n", + "muller2019_model.download_all_layers()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47900e7f-4074-4834-b6e5-30e30b828b8c", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}