Skip to content

Commit

Permalink
Add caching of database + version bump
Browse files Browse the repository at this point in the history
  • Loading branch information
@romainsacchi
romainsacchi committed Sep 17, 2023
1 parent 9dd370f commit 39c1ce1
Show file tree
Hide file tree
Showing 4 changed files with 155 additions and 36 deletions.
152 changes: 119 additions & 33 deletions dev/unfold_test.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1378,29 +1378,54 @@
"`update_all()` will skip the following steps:\n",
"update_two_wheelers(), update_cars(), and update_buses()\n",
"If you want to update these steps, please run them separately afterwards.\n",
"Extracted 1 worksheets in 4.78 seconds\n",
"Extracted 1 worksheets in 10.53 seconds\n",
"Migrating to 3.8 first\n",
"Applying strategy: migrate_datasets\n",
"Applying strategy: migrate_exchanges\n",
"Applying strategy: migrate_datasets\n",
"Applying strategy: migrate_exchanges\n",
"Applying strategy: migrate_datasets\n",
"Applying strategy: migrate_exchanges\n",
"Vehicle fleet data is not available before 2015. Hence, 2015 is used as fleet year.\n",
"Done!\n",
"Incorrect market share for market for hydrogen, gaseous in BRA\n",
"Incorrect market share for market for hydrogen, gaseous in CAN\n",
"Incorrect market share for market for hydrogen, gaseous in CEU\n",
"Incorrect market share for market for hydrogen, gaseous in CHN\n",
"Incorrect market share for market for hydrogen, gaseous in EAF\n",
"Incorrect market share for market for hydrogen, gaseous in INDIA\n",
"Incorrect market share for market for hydrogen, gaseous in INDO\n",
"Incorrect market share for market for hydrogen, gaseous in JAP\n",
"Incorrect market share for market for hydrogen, gaseous in KOR\n",
"Incorrect market share for market for hydrogen, gaseous in ME\n",
"Incorrect market share for market for hydrogen, gaseous in MEX\n",
"Incorrect market share for market for hydrogen, gaseous in NAF\n",
"Incorrect market share for market for hydrogen, gaseous in OCE\n",
"Incorrect market share for market for hydrogen, gaseous in RCAM\n",
"Incorrect market share for market for hydrogen, gaseous in RSAF\n",
"Incorrect market share for market for hydrogen, gaseous in RSAM\n",
"Incorrect market share for market for hydrogen, gaseous in RSAS\n",
"Incorrect market share for market for hydrogen, gaseous in RUS\n",
"Incorrect market share for market for hydrogen, gaseous in SAF\n",
"Incorrect market share for market for hydrogen, gaseous in SEAS\n",
"Incorrect market share for market for hydrogen, gaseous in STAN\n",
"Incorrect market share for market for hydrogen, gaseous in TUR\n",
"Incorrect market share for market for hydrogen, gaseous in UKR\n",
"Incorrect market share for market for hydrogen, gaseous in USA\n",
"Incorrect market share for market for hydrogen, gaseous in WAF\n",
"Incorrect market share for market for hydrogen, gaseous in WEU\n",
"Done!\n",
"Extracted 1 worksheets in 5.36 seconds\n",
"Extracted 1 worksheets in 13.96 seconds\n",
"Migrating to 3.8 first\n",
"Applying strategy: migrate_datasets\n",
"Applying strategy: migrate_exchanges\n",
"Applying strategy: migrate_datasets\n",
"Applying strategy: migrate_exchanges\n",
"Applying strategy: migrate_datasets\n",
"Applying strategy: migrate_exchanges\n",
"Vehicle fleet data is not available before 2015. Hence, 2015 is used as fleet year.\n",
"Vehicle fleet data is not available beyond 2050. Hence, 2050 is used as fleet year.\n",
"Done!\n",
"Done!\n",
"Extracted 1 worksheets in 4.80 seconds\n",
"Extracted 1 worksheets in 8.37 seconds\n",
"Migrating to 3.8 first\n",
"Applying strategy: migrate_datasets\n",
"Applying strategy: migrate_exchanges\n",
Expand All @@ -1409,41 +1434,92 @@
"Applying strategy: migrate_datasets\n",
"Applying strategy: migrate_exchanges\n",
"Done!\n",
"Incorrect market share for market for hydrogen, gaseous in BRA\n",
"Incorrect market share for market for hydrogen, gaseous in CAN\n",
"Incorrect market share for market for hydrogen, gaseous in CEU\n",
"Incorrect market share for market for hydrogen, gaseous in CHN\n",
"Incorrect market share for market for hydrogen, gaseous in EAF\n",
"Incorrect market share for market for hydrogen, gaseous in INDIA\n",
"Incorrect market share for market for hydrogen, gaseous in INDO\n",
"Incorrect market share for market for hydrogen, gaseous in JAP\n",
"Incorrect market share for market for hydrogen, gaseous in KOR\n",
"Incorrect market share for market for hydrogen, gaseous in ME\n",
"Incorrect market share for market for hydrogen, gaseous in MEX\n",
"Incorrect market share for market for hydrogen, gaseous in NAF\n",
"Incorrect market share for market for hydrogen, gaseous in OCE\n",
"Incorrect market share for market for hydrogen, gaseous in RCAM\n",
"Incorrect market share for market for hydrogen, gaseous in RSAF\n",
"Incorrect market share for market for hydrogen, gaseous in RSAM\n",
"Incorrect market share for market for hydrogen, gaseous in RSAS\n",
"Incorrect market share for market for hydrogen, gaseous in RUS\n",
"Incorrect market share for market for hydrogen, gaseous in SAF\n",
"Incorrect market share for market for hydrogen, gaseous in SEAS\n",
"Incorrect market share for market for hydrogen, gaseous in STAN\n",
"Incorrect market share for market for hydrogen, gaseous in TUR\n",
"Incorrect market share for market for hydrogen, gaseous in UKR\n",
"Incorrect market share for market for hydrogen, gaseous in USA\n",
"Incorrect market share for market for hydrogen, gaseous in WAF\n",
"Incorrect market share for market for hydrogen, gaseous in WEU\n",
"Done!\n",
"Extracted 1 worksheets in 8.96 seconds\n",
"Migrating to 3.8 first\n",
"Applying strategy: migrate_datasets\n",
"Applying strategy: migrate_exchanges\n",
"Applying strategy: migrate_datasets\n",
"Applying strategy: migrate_exchanges\n",
"Applying strategy: migrate_datasets\n",
"Applying strategy: migrate_exchanges\n",
"Vehicle fleet data is not available beyond 2050. Hence, 2050 is used as fleet year.\n",
"Done!\n",
"\n",
"Building scenario factor file...\n",
"Number of new activities: 8617\n",
"Data package saved at /Users/romain/GitHub/unfold/dev/export/datapackage/datapackage_2023-09-16.zip\n",
"Generate scenario report.\n",
"Report saved under /Users/romain/GitHub/unfold/dev/export/scenario_report.\n",
"Generate change report.\n",
"Report saved under /Users/romain/GitHub/unfold/dev.\n"
"Done!\n",
"Extracted 1 worksheets in 7.44 seconds\n",
"Migrating to 3.8 first\n",
"Applying strategy: migrate_datasets\n",
"Applying strategy: migrate_exchanges\n",
"Applying strategy: migrate_datasets\n",
"Applying strategy: migrate_exchanges\n",
"Applying strategy: migrate_datasets\n",
"Applying strategy: migrate_exchanges\n",
"Done!\n",
"Done!\n",
"Extracted 1 worksheets in 12.55 seconds\n",
"Migrating to 3.8 first\n",
"Applying strategy: migrate_datasets\n",
"Applying strategy: migrate_exchanges\n",
"Applying strategy: migrate_datasets\n",
"Applying strategy: migrate_exchanges\n",
"Applying strategy: migrate_datasets\n",
"Applying strategy: migrate_exchanges\n",
"Vehicle fleet data is not available beyond 2050. Hence, 2050 is used as fleet year.\n",
"Done!\n",
"Done!\n",
"Done!\n",
"\n"
]
}
],
"source": [
"scenarios = [\n",
" {\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2005},\n",
" {\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2010},\n",
" {\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2015},\n",
" #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2020},\n",
" #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2025},\n",
" #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2030},\n",
" #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2035},\n",
" #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2040},\n",
" #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2045},\n",
" #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2050},\n",
" #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2055},\n",
" #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2060},\n",
" #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2070},\n",
" ##{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2065},\n",
" #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2075},\n",
" #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2085},\n",
" #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2090},\n",
" #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2080},\n",
" #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2095},\n",
" #{\"model\": \"remind\", \"pathway\":\"SSP2-Base\", \"year\": 2100},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2005},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2010},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2015},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2020},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2025},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2030},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2035},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2040},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2045},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2050},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2055},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2060},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2070},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2065},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2075},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2085},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2090},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2080},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2095},\n",
" {\"model\": \"image\", \"pathway\":\"SSP2-Base\", \"year\": 2100},\n",
"]\n",
"\n",
"ndb = NewDatabase(\n",
Expand All @@ -1453,6 +1529,16 @@
" key='tUePmX_S5B8ieZkkM7WUU2CnO8SmShwmAeWK9x2rTFo=', \n",
")\n",
"ndb.update_all()\n",
"#ndb.write_datapackage()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7da1126f-c4db-4b8f-82d9-69a9818f1a10",
"metadata": {},
"outputs": [],
"source": [
"ndb.write_datapackage()"
]
},
Expand Down
2 changes: 1 addition & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def package_files(directory):

setup(
name="unfold",
version="1.1.0",
version="1.1.1",
python_requires=">=3.9",
packages=packages,
author="Romain Sacchi <romain.sacchi@psi.ch>",
Expand Down
2 changes: 1 addition & 1 deletion unfold/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
__all__ = ("Unfold", "Fold")
__version__ = (1, 1, 0)
__version__ = (1, 1, 1)

from .fold import Fold
from .unfold import Unfold
35 changes: 34 additions & 1 deletion unfold/unfold.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from collections import defaultdict
from pathlib import Path
from typing import List, Union
import pickle

import bw2data
import bw2io
Expand Down Expand Up @@ -39,6 +40,7 @@
from .export import UnfoldExporter
from .utils import HiddenPrints

DIR_CACHED_DB = Path(__file__).parent / "cached_databases"

def _c(value):
"""Converts zero to one."""
Expand Down Expand Up @@ -254,12 +256,43 @@ def extract_source_database(self):
"""

for dependency in self.dependencies:
database = extract_brightway2_databases(dependency["source"])
database = self.check_cached_database(dependency["source"])

self.build_mapping_for_dependencies(database)
if dependency.get("type") == "source":
self.database.extend(database)

def clear_existing_cache(self):
"""Clears the cache folder, except for files which contain __version__ in name.
Useful when updating `premise`
or encountering issues with
inventories.
"""
[
f.unlink()
for f in Path(DIR_CACHED_DB).glob("*")
]

def check_cached_database(self, name) -> list:

# check that directory exists, otherwise create it
Path(DIR_CACHED_DB).mkdir(parents=True, exist_ok=True)

file_name = Path(
DIR_CACHED_DB / f"cached_{name}.pickle"
)

# check that file path leads to an existing file
if file_name.exists():
# return the cached database
return pickle.load(open(file_name, "rb"))

# extract the database, pickle it for next time and return it
print("Cannot find cached database. Will create one now for next time...")
database = extract_brightway2_databases(name)
pickle.dump(database, open(file_name, "wb"))
return database

def clean_imported_inventory(self, data):
"""
Cleans the imported inventory.
Expand Down

0 comments on commit 39c1ce1

Please sign in to comment.