diff --git a/dev/unfold_test.ipynb b/dev/unfold_test.ipynb index a8c7790..d11b988 100644 --- a/dev/unfold_test.ipynb +++ b/dev/unfold_test.ipynb @@ -18,8 +18,7 @@ "metadata": {}, "outputs": [], "source": [ - "import brightway2 as bw\n", - "bw.projects.set_current(\"ei39\")" + "bw2data.projects.set_current(\"ei39\")" ] }, { @@ -29,7 +28,7 @@ "metadata": {}, "outputs": [], "source": [ - "fp = \"/Users/romain/GitHub/unfold/dev/export/datapackage/premise15_remind_SSP2_ei39.zip\"" + "fp = \"/Users/romain/Library/Caches/ActivityBrowser/10012454.zip\"" ] }, { @@ -43,12 +42,22 @@ "output_type": "stream", "text": [ "The data package contains the following scenarios:\n", - "+-----+--------------------------------------------------+-------------+\n", - "| No. | Scenario | Description |\n", - "+-----+--------------------------------------------------+-------------+\n", - "| 0 | ecoinvent_cutoff_3.9_remind_SSP2-PkBudg1150_2050 | remind |\n", - "| 1 | ecoinvent_cutoff_3.9_remind_SSP2-PkBudg500_2030 | image |\n", - "+-----+--------------------------------------------------+-------------+\n", + "+-----+---------------------------+------------------------------------------------------------------------------------------------+\n", + "| No. | Scenario | Description |\n", + "+-----+---------------------------+------------------------------------------------------------------------------------------------+\n", + "| 0 | remind - SSP2-Base - 2020 | Prospective db, based on REMIND, pathway SSP2-BASE, for the year 2020, and external scenario . |\n", + "| 1 | remind - SSP2-Base - 2025 | Prospective db, based on REMIND, pathway SSP2-BASE, for the year 2025, and external scenario . |\n", + "| 2 | remind - SSP2-Base - 2030 | Prospective db, based on REMIND, pathway SSP2-BASE, for the year 2030, and external scenario . |\n", + "| 3 | remind - SSP2-Base - 2035 | Prospective db, based on REMIND, pathway SSP2-BASE, for the year 2035, and external scenario . |\n", + "| 4 | remind - SSP2-Base - 2040 | Prospective db, based on REMIND, pathway SSP2-BASE, for the year 2040, and external scenario . |\n", + "| 5 | remind - SSP2-Base - 2045 | Prospective db, based on REMIND, pathway SSP2-BASE, for the year 2045, and external scenario . |\n", + "| 6 | remind - SSP2-Base - 2050 | Prospective db, based on REMIND, pathway SSP2-BASE, for the year 2050, and external scenario . |\n", + "| 7 | remind - SSP2-Base - 2060 | Prospective db, based on REMIND, pathway SSP2-BASE, for the year 2060, and external scenario . |\n", + "| 8 | remind - SSP2-Base - 2070 | Prospective db, based on REMIND, pathway SSP2-BASE, for the year 2070, and external scenario . |\n", + "| 9 | remind - SSP2-Base - 2080 | Prospective db, based on REMIND, pathway SSP2-BASE, for the year 2080, and external scenario . |\n", + "| 10 | remind - SSP2-Base - 2090 | Prospective db, based on REMIND, pathway SSP2-BASE, for the year 2090, and external scenario . |\n", + "| 11 | remind - SSP2-Base - 2100 | Prospective db, based on REMIND, pathway SSP2-BASE, for the year 2100, and external scenario . |\n", + "+-----+---------------------------+------------------------------------------------------------------------------------------------+\n", "\n", "To unfold all scenarios, `unfold()`.\n", "To unfold a specific scenario, `unfold(scenarios=[1,])`.\n" @@ -56,12 +65,12 @@ } ], "source": [ - "u = Unfold(\"test.zip\")" + "u = Unfold(fp)" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "c24a2f2a-b502-4fb9-9db3-ad5769a55ea7", "metadata": {}, "outputs": [ @@ -70,78 +79,92 @@ "output_type": "stream", "text": [ "The following databases are necessary to unfold the scenarios:\n", - "+-----+----------------------+--------------+---------+\n", - "| No. | Dependency | System model | Version |\n", - "+-----+----------------------+--------------+---------+\n", - "| 1 | ecoinvent 3.9 cutoff | cutoff | 3.9 |\n", - "| 2 | biosphere3 | | |\n", - "+-----+----------------------+--------------+---------+\n", + "+-----+------------+--------------+---------+\n", + "| No. | Dependency | System model | Version |\n", + "+-----+------------+--------------+---------+\n", + "| 1 | ecoinvent | cut-off | 3.9 |\n", + "| 2 | biosphere3 | | |\n", + "+-----+------------+--------------+---------+\n", "\n", "The following databases are available in your project:\n", - "+-----+--------------------------------------------------+\n", - "| No. | Database |\n", - "+-----+--------------------------------------------------+\n", - "| 1 | biosphere3 |\n", - "| 2 | ecoinvent 3.9 cutoff |\n", - "| 3 | ESA LCA External 1.1.8a |\n", - "| 4 | ecoinvent 3.9 consequential |\n", - "| 5 | ecoinvent_cutoff_remind_SSP1-Base_2005 |\n", - "| 6 | ecoinvent_cutoff_image_SSP2-Base_2100 |\n", - "| 7 | ecoinvent_cutoff_remind_SSP5-Base_2050 |\n", - "| 8 | ecoinvent_cutoff_3.9_remind_SSP1-Base_2005 |\n", - "| 9 | ecoinvent_cutoff_3.9_image_SSP2-Base_2100 |\n", - "| 10 | ecoinvent_cutoff_3.9_remind_SSP5-Base_2050 |\n", - "| 11 | test1 |\n", - "| 12 | test2 |\n", - "| 13 | ecoinvent 3.9.1 cutoff |\n", - "| 14 | ecoinvent 3.9.1 consequential |\n", - "| 15 | test_super |\n", - "| 16 | brian |\n", - "| 17 | RTE_2020 |\n", - "| 18 | RTE_2030 |\n", - "| 19 | RTE_2040 |\n", - "| 20 | RTE_2050 |\n", - "| 21 | RTE_2060 |\n", - "| 22 | RTE_2020_M0 |\n", - "| 23 | RTE_2025_M0 |\n", - "| 24 | RTE_2030_M0 |\n", - "| 25 | RTE_2035_M0 |\n", - "| 26 | RTE_2040_M0 |\n", - "| 27 | RTE_2045_M0 |\n", - "| 28 | RTE_2050_M0 |\n", - "| 29 | RTE_2055_M0 |\n", - "| 30 | RTE_2060_M0 |\n", - "| 31 | RTE_2020_sobriety_N3 |\n", - "| 32 | RTE_2025_sobriety_N3 |\n", - "| 33 | RTE_2030_sobriety_N3 |\n", - "| 34 | RTE_2035_sobriety_N3 |\n", - "| 35 | RTE_2040_sobriety_N3 |\n", - "| 36 | RTE_2045_sobriety_N3 |\n", - "| 37 | RTE_2050_sobriety_N3 |\n", - "| 38 | RTE_2055_sobriety_N3 |\n", - "| 39 | RTE_2060_sobriety_N3 |\n", - "| 40 | RTE_2020_ext_N1 |\n", - "| 41 | RTE_2025_ext_N1 |\n", - "| 42 | RTE_2030_ext_N1 |\n", - "| 43 | RTE_2035_ext_N1 |\n", - "| 44 | RTE_2040_ext_N1 |\n", - "| 45 | RTE_2045_ext_N1 |\n", - "| 46 | RTE_2050_ext_N1 |\n", - "| 47 | RTE_2055_ext_N1 |\n", - "| 48 | RTE_2060_ext_N1 |\n", - "| 49 | ecoinvent_cutoff_3.9_remind_SSP2-NPi_2030 |\n", - "| 50 | ecoinvent_cutoff_3.9_remind_SSP2-NPi_2040 |\n", - "| 51 | ecoinvent_cutoff_3.9_remind_SSP2-NPi_2050 |\n", - "| 52 | ecoinvent_cutoff_3.9_remind_SSP2-PkBudg1150_2030 |\n", - "| 53 | ecoinvent_cutoff_3.9_remind_SSP2-PkBudg1150_2040 |\n", - "| 54 | ecoinvent_cutoff_3.9_remind_SSP2-PkBudg1150_2050 |\n", - "| 55 | ecoinvent_cutoff_3.9_remind_SSP2-PkBudg500_2030 |\n", - "| 56 | ecoinvent_cutoff_3.9_remind_SSP2-PkBudg500_2040 |\n", - "| 57 | ecoinvent_cutoff_3.9_remind_SSP2-PkBudg500_2050 |\n", - "| 58 | trucks_bw_ei_39 |\n", - "| 59 | remind - SSP2-Base - 2005 |\n", - "| 60 | remind - SSP2-Base - 2010 |\n", - "+-----+--------------------------------------------------+\n", + "+-----+------------------------------------------------------------------------------------+\n", + "| No. | Database |\n", + "+-----+------------------------------------------------------------------------------------+\n", + "| 1 | biosphere3 |\n", + "| 2 | ecoinvent 3.9.1 cutoff |\n", + "| 3 | ecoinvent EN15804 3.9.1 |\n", + "| 4 | remind - SSP2-Base - 2005 |\n", + "| 5 | remind - SSP2-Base - 2010 |\n", + "| 6 | remind - SSP2-Base - 2015 |\n", + "| 7 | stationary battery |\n", + "| 8 | test1 |\n", + "| 9 | test2 |\n", + "| 10 | test_super |\n", + "| 11 | test |\n", + "| 12 | PV 2050 - eff |\n", + "| 13 | PV 2050 - eff - lifetime |\n", + "| 14 | PV + storage |\n", + "| 15 | ecoinvent 3.9.1 consequential |\n", + "| 16 | ecoinvent_consequential_3.9_remind_SSP1-Base_2050 |\n", + "| 17 | ecoinvent_consequential_3.9_remind_SSP2-Base_2050 |\n", + "| 18 | ecoinvent_consequential_3.9_remind_SSP2-NPi_2020 |\n", + "| 19 | test_vanadium |\n", + "| 20 | vanadium-redox-flow-battery |\n", + "| 21 | test_storage1 |\n", + "| 22 | test_storage2 |\n", + "| 23 | test_tiam |\n", + "| 24 | image - SSP2-Base - 2020 |\n", + "| 25 | image - SSP2-Base - 2020 - energy-perspective-2050-switzerland - Business As Usual |\n", + "| 26 | image - SSP2-Base - 2050 - energy-perspective-2050-switzerland - Business As Usual |\n", + "| 27 | image - SSP2-RCP26 - 2050 - energy-perspective-2050-switzerland - ZERO Basis |\n", + "| 28 | energy-perspective-2050-switzerland |\n", + "| 29 | cobalt-perspective-2050 |\n", + "| 30 | ammonia-prospective-scenarios |\n", + "| 31 | PV 2035 |\n", + "| 32 | PV 2050 |\n", + "| 33 | elec |\n", + "| 34 | elec + steel |\n", + "| 35 | elec + steel + cement |\n", + "| 36 | bread 2030 |\n", + "| 37 | bread 2050 |\n", + "| 38 | BAU - 2035 |\n", + "| 39 | BAU - 2050 |\n", + "| 40 | CA - 2035 |\n", + "| 41 | CA - 2050 |\n", + "| 42 | remind - SSP2-NDC - 2030 |\n", + "| 43 | remind - SSP2-NDC - 2080 |\n", + "| 44 | super_db_2023-10-10 |\n", + "| 45 | ecoinvent_cutoff_3.9_image_SSP2-Base_2050 |\n", + "| 46 | ecoinvent_cutoff_3.9_remind_SSP2-PkBudg500_2050 |\n", + "| 47 | ecoinvent_cutoff_3.9_remind_SSP2-Base_2030 |\n", + "| 48 | ecoinvent_cutoff_3.9_remind_SSP2-Base_2050 |\n", + "| 49 | ecoinvent_cutoff_3.9_remind_SSP2-PkBudg500_2030 |\n", + "| 50 | remind - SSP2-Base - 2020 |\n", + "| 51 | remind - SSP2-Base - 2050 |\n", + "| 52 | SP test |\n", + "| 53 | ei39-remind-SSP2-test |\n", + "| 54 | ei39-remind-SSP2 |\n", + "| 55 | test3 |\n", + "| 56 | test4 |\n", + "| 57 | test5 |\n", + "| 58 | test6 |\n", + "| 59 | test7 |\n", + "| 60 | test8 |\n", + "| 61 | test9 |\n", + "| 62 | test10 |\n", + "| 63 | test13 |\n", + "| 64 | test14 |\n", + "| 65 | test15 |\n", + "| 66 | test16 |\n", + "| 67 | test17 |\n", + "| 68 | test18 |\n", + "| 69 | SDF test |\n", + "| 70 | SP remind |\n", + "| 71 | SP test rebecca |\n", + "| 72 | test ulla |\n", + "| 73 | Power-2-heat |\n", + "| 74 | remind SSP2-Base |\n", + "+-----+------------------------------------------------------------------------------------+\n", "\n" ] }, @@ -149,137 +172,27 @@ "name": "stdin", "output_type": "stream", "text": [ - "Indicate the database number for dependency 1: 13\n", + "Indicate the database number for dependency 1: 2\n", "Indicate the database number for dependency 2: 1\n" ] }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Getting activity data\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|█████████████████████████████████| 21238/21238 [00:00<00:00, 289392.97it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Adding exchange data to activities\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|████████████████████████████████| 674593/674593 [00:23<00:00, 28460.44it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Filling out exchange data\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████████████████████████████| 21238/21238 [00:01<00:00, 11317.57it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Getting activity data\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████| 4733/4733 [00:00<00:00, 283825.98it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Adding exchange data to activities\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "0it [00:00, ?it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Filling out exchange data\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|██████████████████████████████████| 4733/4733 [00:00<00:00, 3823505.55it/s]\n" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ "Extracting additional inventories...\n", - "Generating database for scenario ecoinvent_cutoff_3.9_remind_SSP2-PkBudg1150_2050...\n", - "Generating database for scenario ecoinvent_cutoff_3.9_remind_SSP2-PkBudg500_2030...\n", - "Writing database for scenario ecoinvent_cutoff_3.9_remind_SSP2-PkBudg1150_2050...\n", - "Database ecoinvent_cutoff_3.9_remind_SSP2-PkBudg1150_2050 already exists: it will be overwritten.\n", + "Writing scenario difference file...\n", + "Generating superstructure database...\n", + "Scenario difference file exported to /Users/romain/GitHub/unfold/dev/SDF ecoinvent 3.9 remind SSP2-Base.csv\n", + "\n", + "Writing superstructure database...\n", + "Database remind SSP2-Base already exists: it will be overwritten.\n", "Vacuuming database \n" ] - }, - { - "ename": "OperationalError", - "evalue": "database table is locked", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mOperationalError\u001b[0m Traceback (most recent call last)", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/unfold/lib/python3.10/site-packages/peewee.py:3237\u001b[0m, in \u001b[0;36mDatabase.execute_sql\u001b[0;34m(self, sql, params, commit)\u001b[0m\n\u001b[1;32m 3236\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 3237\u001b[0m \u001b[43mcursor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43msql\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 3238\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:\n", - "\u001b[0;31mOperationalError\u001b[0m: database table is locked", - "\nDuring handling of the above exception, another exception occurred:\n", - "\u001b[0;31mOperationalError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn [6], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mu\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43munfold\u001b[49m\u001b[43m(\u001b[49m\u001b[43mscenarios\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/GitHub/unfold/unfold/unfold.py:1024\u001b[0m, in \u001b[0;36mUnfold.unfold\u001b[0;34m(self, scenarios, dependencies, superstructure)\u001b[0m\n\u001b[1;32m 1021\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mformat_superstructure_dataframe()\n\u001b[1;32m 1022\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdatabase \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgenerate_superstructure_database()\n\u001b[0;32m-> 1024\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrite\u001b[49m\u001b[43m(\u001b[49m\u001b[43msuperstructure\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msuperstructure\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/GitHub/unfold/unfold/unfold.py:1045\u001b[0m, in \u001b[0;36mUnfold.write\u001b[0;34m(self, superstructure)\u001b[0m\n\u001b[1;32m 1043\u001b[0m correct_fields_format(database, scenario)\n\u001b[1;32m 1044\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mWriting database for scenario \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mscenario\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m...\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1045\u001b[0m \u001b[43mUnfoldExporter\u001b[49m\u001b[43m(\u001b[49m\u001b[43mscenario\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdatabase\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrite_database\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1047\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 1048\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n", - "File \u001b[0;32m~/GitHub/unfold/unfold/export.py:28\u001b[0m, in \u001b[0;36mUnfoldExporter.write_database\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdb_name \u001b[38;5;129;01min\u001b[39;00m databases:\n\u001b[1;32m 25\u001b[0m \u001b[38;5;28mprint\u001b[39m(\n\u001b[1;32m 26\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDatabase \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdb_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124malready exists: it will be overwritten.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 27\u001b[0m )\n\u001b[0;32m---> 28\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrite_database\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/unfold/lib/python3.10/site-packages/bw2io/importers/base_lci.py:269\u001b[0m, in \u001b[0;36mLCIImporter.write_database\u001b[0;34m(self, data, delete_existing, backend, activate_parameters, **kwargs)\u001b[0m\n\u001b[1;32m 266\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwrite_database_parameters(activate_parameters, delete_existing)\n\u001b[1;32m 268\u001b[0m existing\u001b[38;5;241m.\u001b[39mupdate(data)\n\u001b[0;32m--> 269\u001b[0m \u001b[43mdb\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrite\u001b[49m\u001b[43m(\u001b[49m\u001b[43mexisting\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 271\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m activate_parameters:\n\u001b[1;32m 272\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_write_activity_parameters(activity_parameters)\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/unfold/lib/python3.10/site-packages/bw2data/project.py:358\u001b[0m, in \u001b[0;36mwritable_project\u001b[0;34m(wrapped, instance, args, kwargs)\u001b[0m\n\u001b[1;32m 356\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m projects\u001b[38;5;241m.\u001b[39mread_only:\n\u001b[1;32m 357\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m ReadOnlyProject(READ_ONLY_PROJECT)\n\u001b[0;32m--> 358\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mwrapped\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/unfold/lib/python3.10/site-packages/bw2data/backends/peewee/database.py:260\u001b[0m, in \u001b[0;36mSQLiteBackend.write\u001b[0;34m(self, data, process)\u001b[0m\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m data:\n\u001b[1;32m 259\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 260\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_efficient_write_many_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 261\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m:\n\u001b[1;32m 262\u001b[0m \u001b[38;5;66;03m# Purge all data from database, then reraise\u001b[39;00m\n\u001b[1;32m 263\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdelete(warn\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/unfold/lib/python3.10/site-packages/bw2data/backends/peewee/database.py:189\u001b[0m, in \u001b[0;36mSQLiteBackend._efficient_write_many_data\u001b[0;34m(self, data, indices)\u001b[0m\n\u001b[1;32m 187\u001b[0m be_complicated \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(data) \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m100\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m indices\n\u001b[1;32m 188\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m be_complicated:\n\u001b[0;32m--> 189\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_drop_indices\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 190\u001b[0m sqlite3_lci_db\u001b[38;5;241m.\u001b[39mdb\u001b[38;5;241m.\u001b[39mautocommit \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 191\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/unfold/lib/python3.10/site-packages/bw2data/backends/peewee/database.py:143\u001b[0m, in \u001b[0;36mSQLiteBackend._drop_indices\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 141\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_drop_indices\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 142\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m sqlite3_lci_db\u001b[38;5;241m.\u001b[39mtransaction():\n\u001b[0;32m--> 143\u001b[0m \u001b[43msqlite3_lci_db\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute_sql\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mDROP INDEX IF EXISTS \u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mactivitydataset_key\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 144\u001b[0m sqlite3_lci_db\u001b[38;5;241m.\u001b[39mexecute_sql(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mDROP INDEX IF EXISTS \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexchangedataset_input\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 145\u001b[0m sqlite3_lci_db\u001b[38;5;241m.\u001b[39mexecute_sql(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mDROP INDEX IF EXISTS \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mexchangedataset_output\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m'\u001b[39m)\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/unfold/lib/python3.10/site-packages/bw2data/sqlite.py:50\u001b[0m, in \u001b[0;36mSubstitutableDatabase.execute_sql\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mexecute_sql\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m---> 50\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdb\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute_sql\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/unfold/lib/python3.10/site-packages/peewee.py:3234\u001b[0m, in \u001b[0;36mDatabase.execute_sql\u001b[0;34m(self, sql, params, commit)\u001b[0m\n\u001b[1;32m 3231\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 3232\u001b[0m commit \u001b[38;5;241m=\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m sql[:\u001b[38;5;241m6\u001b[39m]\u001b[38;5;241m.\u001b[39mlower()\u001b[38;5;241m.\u001b[39mstartswith(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mselect\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m-> 3234\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m __exception_wrapper__:\n\u001b[1;32m 3235\u001b[0m cursor \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcursor(commit)\n\u001b[1;32m 3236\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/unfold/lib/python3.10/site-packages/peewee.py:3010\u001b[0m, in \u001b[0;36mExceptionWrapper.__exit__\u001b[0;34m(self, exc_type, exc_value, traceback)\u001b[0m\n\u001b[1;32m 3008\u001b[0m new_type \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexceptions[exc_type\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m]\n\u001b[1;32m 3009\u001b[0m exc_args \u001b[38;5;241m=\u001b[39m exc_value\u001b[38;5;241m.\u001b[39margs\n\u001b[0;32m-> 3010\u001b[0m \u001b[43mreraise\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnew_type\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnew_type\u001b[49m\u001b[43m(\u001b[49m\u001b[43mexc_value\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mexc_args\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtraceback\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/unfold/lib/python3.10/site-packages/peewee.py:192\u001b[0m, in \u001b[0;36mreraise\u001b[0;34m(tp, value, tb)\u001b[0m\n\u001b[1;32m 190\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mreraise\u001b[39m(tp, value, tb\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[1;32m 191\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m value\u001b[38;5;241m.\u001b[39m__traceback__ \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m tb:\n\u001b[0;32m--> 192\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m value\u001b[38;5;241m.\u001b[39mwith_traceback(tb)\n\u001b[1;32m 193\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m value\n", - "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/unfold/lib/python3.10/site-packages/peewee.py:3237\u001b[0m, in \u001b[0;36mDatabase.execute_sql\u001b[0;34m(self, sql, params, commit)\u001b[0m\n\u001b[1;32m 3235\u001b[0m cursor \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcursor(commit)\n\u001b[1;32m 3236\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 3237\u001b[0m \u001b[43mcursor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43msql\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01mor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 3238\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:\n\u001b[1;32m 3239\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mautorollback \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39min_transaction():\n", - "\u001b[0;31mOperationalError\u001b[0m: database table is locked" - ] } ], "source": [ - "u.unfold(scenarios=[0, 1])" + "u.unfold(scenarios=[0, 1], superstructure=True)" ] }, { diff --git a/unfold/data/outdated_units.yaml b/unfold/data/outdated_units.yaml new file mode 100644 index 0000000..cec4e45 --- /dev/null +++ b/unfold/data/outdated_units.yaml @@ -0,0 +1,46 @@ +# some units may be changed from one version of bw2io to another +--- + +a: year +bq: Becquerel +g: gram +gj: gigajoule +h: hour +ha: hectare +hr: hour +kbq: kilo Becquerel +kg: kilogram +kgkm: kilogram kilometer +kg*day: kilogram day +kilogram day: kg*day +km: kilometer +kj: kilojoule +kwh: kilowatt hour +l: litre +lu: livestock unit +m: meter +m*year: meter-year +m2: square meter +m2*year: square meter-year +m2a: square meter-year +m2y: square meter-year +m3: cubic meter +m3*year: cubic meter-year +m3a: cubic meter-year +m3y: cubic meter-year +ma: meter-year +metric ton*km: ton kilometer +mj: megajoule +my: meter-year +nm3: normal cubic meter +sm3: standard cubic meter +p: unit +personkm: person kilometer +person*km: person kilometer +pkm: person kilometer +tonnes: ton +tkm: ton kilometer +vkm: vehicle kilometer +kg sw: kilogram separative work unit +km*year: kilometer-year +wh: watt hour diff --git a/unfold/data_cleaning.py b/unfold/data_cleaning.py index 4b2e6f9..f2716e7 100644 --- a/unfold/data_cleaning.py +++ b/unfold/data_cleaning.py @@ -9,6 +9,7 @@ DATA_DIR = Path(__file__).resolve().parent / "data" OUTDATED_FLOWS = DATA_DIR / "outdated_flows.yaml" +OUTDATED_UNITS = DATA_DIR / "outdated_units.yaml" def get_outdated_flows() -> dict: @@ -21,6 +22,16 @@ def get_outdated_flows() -> dict: return flows +def get_outdated_units() -> dict: + """ + Retrieve a list of outdated flows from the outdated flows file. + """ + + with open(OUTDATED_UNITS, "r", encoding="utf-8") as stream: + flows = yaml.safe_load(stream) + + return flows + def remove_missing_fields(data: List[dict]) -> List[dict]: """ diff --git a/unfold/unfold.py b/unfold/unfold.py index e8f8e12..f421fe1 100644 --- a/unfold/unfold.py +++ b/unfold/unfold.py @@ -35,6 +35,7 @@ correct_fields_format, get_list_of_unique_datasets, get_outdated_flows, + get_outdated_units, remove_categories_for_technosphere_flows, remove_missing_fields, ) @@ -144,6 +145,7 @@ def __init__(self, path: Union[str, Path]): self.acts_indices = None self.reversed_acts_indices = None self.outdated_flows = get_outdated_flows() + self.outdated_units = get_outdated_units() self.path = path self.package = Package(self.path) self.dependencies = self.package.descriptor["dependencies"] @@ -489,6 +491,26 @@ def fix_key(self, key: tuple) -> tuple: if key in self.dependency_mapping: return self.dependency_mapping[key] else: + # try first by searching for alternative units + if key[4] in self.outdated_units: + print(f"found alternative unit: {key[4]}. Changing to {self.outdated_units[key[4]]}") + if ( + key[0], + key[1], + key[2], + key[3], + self.outdated_units[key[4]], + key[5], + ) in self.dependency_mapping: + return self.dependency_mapping[( + key[0], + key[1], + key[2], + key[3], + self.outdated_units[key[4]], + key[5], + )] + correct_id = self.find_correct_id(key) if correct_id is None: print(f"Could not find key: {key}") @@ -623,6 +645,19 @@ def write_scaling_factors_in_matrix( None, ) + if consumer_idx is None: + consumer_idx = self.reversed_acts_indices.get( + ( + c_name, + c_prod, + None, + c_loc, + self.outdated_units.get(c_unit, c_unit), + "production", + ), + None, + ) + # Look up the index of the supplier activity in the reversed activities index. supplier_id = ( s_name, @@ -637,6 +672,19 @@ def write_scaling_factors_in_matrix( None, ) + if supplier_idx is None: + supplier_idx = self.reversed_acts_indices.get( + ( + s_name, + s_prod, + s_cat, + s_loc, + self.outdated_units.get(s_unit, s_unit), + s_type, + ), + None, + ) + # Multiply the appropriate element of the matrix by the scaling factor for the given scenario. # Use the lambda function defined above to avoid multiplying by zero. if supplier_idx is not None and consumer_idx is not None: @@ -1007,18 +1055,37 @@ def format_superstructure_dataframe(self) -> None: # Get the indices of the consumer and supplier activities # in the reversed_act_indices dictionary - consumer_idx = self.reversed_acts_indices[ - ( + consumer_key = ( + c_name, + c_prod, + None, + c_loc, + c_unit, + "production", + ) + if consumer_key in self.reversed_acts_indices: + consumer_idx = self.reversed_acts_indices[ + consumer_key + ] + else: + consumer_key = ( c_name, c_prod, None, c_loc, - c_unit, + self.outdated_units.get(c_unit, c_unit), "production", ) - ] - supplier_id = ( + if consumer_key in self.reversed_acts_indices: + consumer_idx = self.reversed_acts_indices[ + consumer_key + ] + else: + print("not found key for consumer ", consumer_key) + consumer_idx = 0 + + supplier_key = ( s_name, s_prod, s_cat, @@ -1027,7 +1094,7 @@ def format_superstructure_dataframe(self) -> None: s_type, ) - if supplier_id not in self.reversed_acts_indices: + if supplier_key not in self.reversed_acts_indices: try: ( s_name, @@ -1036,7 +1103,7 @@ def format_superstructure_dataframe(self) -> None: s_cat, ) = self.find_correct_id((s_name, s_prod, s_loc, s_cat)) - supplier_id = ( + supplier_key = ( s_name, s_prod, s_cat, @@ -1045,9 +1112,20 @@ def format_superstructure_dataframe(self) -> None: s_type, ) except TypeError: - print("not found key for", supplier_id) - supplier_idx = self.reversed_acts_indices[supplier_id] + supplier_key = ( + s_name, + s_prod, + s_cat, + s_loc, + self.outdated_units.get(s_unit, s_unit), + s_type, + ) + + if supplier_key not in self.reversed_acts_indices: + print("not found key for supplier", supplier_key) + + supplier_idx = self.reversed_acts_indices[supplier_key] # Update the factor values for each scenario # by multiplying with the corresponding matrix value