diff --git a/dev/unfold_test.ipynb b/dev/unfold_test.ipynb index 396141e..a0a316d 100644 --- a/dev/unfold_test.ipynb +++ b/dev/unfold_test.ipynb @@ -1378,7 +1378,7 @@ "`update_all()` will skip the following steps:\n", "update_two_wheelers(), update_cars(), and update_buses()\n", "If you want to update these steps, please run them separately afterwards.\n", - "Extracted 1 worksheets in 4.78 seconds\n", + "Extracted 1 worksheets in 10.53 seconds\n", "Migrating to 3.8 first\n", "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", @@ -1386,10 +1386,35 @@ "Applying strategy: migrate_exchanges\n", "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", - "Vehicle fleet data is not available before 2015. Hence, 2015 is used as fleet year.\n", "Done!\n", + "Incorrect market share for market for hydrogen, gaseous in BRA\n", + "Incorrect market share for market for hydrogen, gaseous in CAN\n", + "Incorrect market share for market for hydrogen, gaseous in CEU\n", + "Incorrect market share for market for hydrogen, gaseous in CHN\n", + "Incorrect market share for market for hydrogen, gaseous in EAF\n", + "Incorrect market share for market for hydrogen, gaseous in INDIA\n", + "Incorrect market share for market for hydrogen, gaseous in INDO\n", + "Incorrect market share for market for hydrogen, gaseous in JAP\n", + "Incorrect market share for market for hydrogen, gaseous in KOR\n", + "Incorrect market share for market for hydrogen, gaseous in ME\n", + "Incorrect market share for market for hydrogen, gaseous in MEX\n", + "Incorrect market share for market for hydrogen, gaseous in NAF\n", + "Incorrect market share for market for hydrogen, gaseous in OCE\n", + "Incorrect market share for market for hydrogen, gaseous in RCAM\n", + "Incorrect market share for market for hydrogen, gaseous in RSAF\n", + "Incorrect market share for market for hydrogen, gaseous in RSAM\n", + "Incorrect market share for market for hydrogen, gaseous in RSAS\n", + "Incorrect market share for market for hydrogen, gaseous in RUS\n", + "Incorrect market share for market for hydrogen, gaseous in SAF\n", + "Incorrect market share for market for hydrogen, gaseous in SEAS\n", + "Incorrect market share for market for hydrogen, gaseous in STAN\n", + "Incorrect market share for market for hydrogen, gaseous in TUR\n", + "Incorrect market share for market for hydrogen, gaseous in UKR\n", + "Incorrect market share for market for hydrogen, gaseous in USA\n", + "Incorrect market share for market for hydrogen, gaseous in WAF\n", + "Incorrect market share for market for hydrogen, gaseous in WEU\n", "Done!\n", - "Extracted 1 worksheets in 5.36 seconds\n", + "Extracted 1 worksheets in 13.96 seconds\n", "Migrating to 3.8 first\n", "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", @@ -1397,10 +1422,10 @@ "Applying strategy: migrate_exchanges\n", "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", - "Vehicle fleet data is not available before 2015. Hence, 2015 is used as fleet year.\n", + "Vehicle fleet data is not available beyond 2050. Hence, 2050 is used as fleet year.\n", "Done!\n", "Done!\n", - "Extracted 1 worksheets in 4.80 seconds\n", + "Extracted 1 worksheets in 8.37 seconds\n", "Migrating to 3.8 first\n", "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", @@ -1409,41 +1434,92 @@ "Applying strategy: migrate_datasets\n", "Applying strategy: migrate_exchanges\n", "Done!\n", + "Incorrect market share for market for hydrogen, gaseous in BRA\n", + "Incorrect market share for market for hydrogen, gaseous in CAN\n", + "Incorrect market share for market for hydrogen, gaseous in CEU\n", + "Incorrect market share for market for hydrogen, gaseous in CHN\n", + "Incorrect market share for market for hydrogen, gaseous in EAF\n", + "Incorrect market share for market for hydrogen, gaseous in INDIA\n", + "Incorrect market share for market for hydrogen, gaseous in INDO\n", + "Incorrect market share for market for hydrogen, gaseous in JAP\n", + "Incorrect market share for market for hydrogen, gaseous in KOR\n", + "Incorrect market share for market for hydrogen, gaseous in ME\n", + "Incorrect market share for market for hydrogen, gaseous in MEX\n", + "Incorrect market share for market for hydrogen, gaseous in NAF\n", + "Incorrect market share for market for hydrogen, gaseous in OCE\n", + "Incorrect market share for market for hydrogen, gaseous in RCAM\n", + "Incorrect market share for market for hydrogen, gaseous in RSAF\n", + "Incorrect market share for market for hydrogen, gaseous in RSAM\n", + "Incorrect market share for market for hydrogen, gaseous in RSAS\n", + "Incorrect market share for market for hydrogen, gaseous in RUS\n", + "Incorrect market share for market for hydrogen, gaseous in SAF\n", + "Incorrect market share for market for hydrogen, gaseous in SEAS\n", + "Incorrect market share for market for hydrogen, gaseous in STAN\n", + "Incorrect market share for market for hydrogen, gaseous in TUR\n", + "Incorrect market share for market for hydrogen, gaseous in UKR\n", + "Incorrect market share for market for hydrogen, gaseous in USA\n", + "Incorrect market share for market for hydrogen, gaseous in WAF\n", + "Incorrect market share for market for hydrogen, gaseous in WEU\n", "Done!\n", + "Extracted 1 worksheets in 8.96 seconds\n", + "Migrating to 3.8 first\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Vehicle fleet data is not available beyond 2050. Hence, 2050 is used as fleet year.\n", "Done!\n", - "\n", - "Building scenario factor file...\n", - "Number of new activities: 8617\n", - "Data package saved at /Users/romain/GitHub/unfold/dev/export/datapackage/datapackage_2023-09-16.zip\n", - "Generate scenario report.\n", - "Report saved under /Users/romain/GitHub/unfold/dev/export/scenario_report.\n", - "Generate change report.\n", - "Report saved under /Users/romain/GitHub/unfold/dev.\n" + "Done!\n", + "Extracted 1 worksheets in 7.44 seconds\n", + "Migrating to 3.8 first\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Done!\n", + "Done!\n", + "Extracted 1 worksheets in 12.55 seconds\n", + "Migrating to 3.8 first\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Applying strategy: migrate_datasets\n", + "Applying strategy: migrate_exchanges\n", + "Vehicle fleet data is not available beyond 2050. Hence, 2050 is used as fleet year.\n", + "Done!\n", + "Done!\n", + "Done!\n", + "\n" ] } ], "source": [ "scenarios = [\n", - " {\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2005},\n", - " {\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2010},\n", - " {\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2015},\n", - " #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2020},\n", - " #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2025},\n", - " #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2030},\n", - " #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2035},\n", - " #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2040},\n", - " #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2045},\n", - " #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2050},\n", - " #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2055},\n", - " #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2060},\n", - " #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2070},\n", - " ##{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2065},\n", - " #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2075},\n", - " #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2085},\n", - " #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2090},\n", - " #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2080},\n", - " #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2095},\n", - " #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2100},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2005},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2010},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2015},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2020},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2025},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2030},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2035},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2040},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2045},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2050},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2055},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2060},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2070},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2065},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2075},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2085},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2090},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2080},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2095},\n", + " {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2100},\n", "]\n", "\n", "ndb = NewDatabase(\n", @@ -1453,6 +1529,16 @@ " key='tUePmX_S5B8ieZkkM7WUU2CnO8SmShwmAeWK9x2rTFo=', \n", ")\n", "ndb.update_all()\n", + "#ndb.write_datapackage()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7da1126f-c4db-4b8f-82d9-69a9818f1a10", + "metadata": {}, + "outputs": [], + "source": [ "ndb.write_datapackage()" ] }, diff --git a/setup.py b/setup.py index a39a2bd..c58e0d5 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def package_files(directory): setup( name="unfold", - version="1.1.0", + version="1.1.1", python_requires=">=3.9", packages=packages, author="Romain Sacchi ", diff --git a/unfold/__init__.py b/unfold/__init__.py index 2189bc7..952d866 100644 --- a/unfold/__init__.py +++ b/unfold/__init__.py @@ -1,5 +1,5 @@ __all__ = ("Unfold", "Fold") -__version__ = (1, 1, 0) +__version__ = (1, 1, 1) from .fold import Fold from .unfold import Unfold diff --git a/unfold/unfold.py b/unfold/unfold.py index 1f8ee46..62480d0 100644 --- a/unfold/unfold.py +++ b/unfold/unfold.py @@ -8,6 +8,7 @@ from collections import defaultdict from pathlib import Path from typing import List, Union +import pickle import bw2data import bw2io @@ -39,6 +40,7 @@ from .export import UnfoldExporter from .utils import HiddenPrints +DIR_CACHED_DB = Path(__file__).parent / "cached_databases" def _c(value): """Converts zero to one.""" @@ -254,12 +256,43 @@ def extract_source_database(self): """ for dependency in self.dependencies: - database = extract_brightway2_databases(dependency["source"]) + database = self.check_cached_database(dependency["source"]) self.build_mapping_for_dependencies(database) if dependency.get("type") == "source": self.database.extend(database) + def clear_existing_cache(self): + """Clears the cache folder, except for files which contain __version__ in name. + Useful when updating `premise` + or encountering issues with + inventories. + """ + [ + f.unlink() + for f in Path(DIR_CACHED_DB).glob("*") + ] + + def check_cached_database(self, name) -> list: + + # check that directory exists, otherwise create it + Path(DIR_CACHED_DB).mkdir(parents=True, exist_ok=True) + + file_name = Path( + DIR_CACHED_DB / f"cached_{name}.pickle" + ) + + # check that file path leads to an existing file + if file_name.exists(): + # return the cached database + return pickle.load(open(file_name, "rb")) + + # extract the database, pickle it for next time and return it + print("Cannot find cached database. Will create one now for next time...") + database = extract_brightway2_databases(name) + pickle.dump(database, open(file_name, "wb")) + return database + def clean_imported_inventory(self, data): """ Cleans the imported inventory.