diff --git a/Start_Here.ipynb b/Start_Here.ipynb index 55b802ca4..31fa12fbb 100644 --- a/Start_Here.ipynb +++ b/Start_Here.ipynb @@ -2,64 +2,113 @@ "cells": [ { "cell_type": "markdown", - "id": "dcb7135e", - "metadata": {}, "source": [ - "Welcome to the Drawdown Solutions library!\n", - "\n", - "This jupyter notebook is a stub. It needs to be expanded to include additional useful samples.\n", - "\n", - "(A helpful tutorial for jupyter notebooks is [here](https://nbviewer.jupyter.org/github/jupyter/notebook/blob/master/docs/source/examples/Notebook/Notebook%20Basics.ipynb) or [here](https://youtu.be/3C9E2yPBw7s?t=131) if you'd prefer a video introduction.)" - ] + "Welcome to the Drawdown Solutions library!\r\n", + "\r\n", + "This jupyter notebook is a stub. It needs to be expanded to include additional useful samples.\r\n", + "\r\n", + "(A helpful tutorial for jupyter notebooks is [here](https://nbviewer.jupyter.org/github/jupyter/notebook/blob/master/docs/source/examples/Notebook/Notebook%20Basics.ipynb)---or [here](https://youtu.be/3C9E2yPBw7s?t=131) if you'd prefer a video introduction.)" + ], + "metadata": {} }, { "cell_type": "markdown", - "id": "bd0ed19e", - "metadata": {}, "source": [ "# Working with Solutions/Scenarios" - ] + ], + "metadata": {} }, { "cell_type": "code", "execution_count": 1, - "id": "a8344e92", - "metadata": {}, - "outputs": [], "source": [ - "# Import modules we'll use later\n", - "\n", - "from solution import factory\n", - "import pandas as pd\n", + "# Import modules we'll use later\r\n", + "\r\n", + "from solution import factory\r\n", + "import pandas as pd\r\n", "import matplotlib" - ] + ], + "outputs": [], + "metadata": {} }, { "cell_type": "code", - "execution_count": 4, - "id": "7810e5e6", - "metadata": {}, - "outputs": [], + "execution_count": 2, "source": [ - "# Solutions don't really have a data structure of their own. What you get from the factory\n", - "# is a constructor for a _scenario_ and a list of scenario names.\n", - "# Then you can construct one of the scenario objects from that:\n", - "\n", - "# Load the \"Bike Infrastructure\" solution.\n", - "# Every directory in the `solution/` subdirectory is a solution.\n", - "\n", - "(constructor, scenarios) = factory.one_solution_scenarios('bikeinfrastructure')\n", - "bikeinfra_scenario0 = constructor(scenarios[0])" - ] + "# The factory module has several ways of constructing solutions and scenarios.\r\n", + "# The simplest is to get the most recent PDS scenario of a particular type for a particular solution\r\n", + "\r\n", + "bikeinfra_pds2 = factory.solution_pds_type(\"bikeinfrastructure\", \"PDS2\")" + ], + "outputs": [], + "metadata": {} }, { "cell_type": "code", - "execution_count": 9, - "id": "205ff20b", - "metadata": {}, + "execution_count": 3, + "source": [ + "# All the work of calculating carbon impact, capital cost and operating cost for the scenario \r\n", + "# was done by that constructure. Here, for example, is the carbon impact. (In this case, we\r\n", + "# only have global data, and only modeled out to 2050)\r\n", + "\r\n", + "bi_co2e = bikeinfra_pds2.c2.co2eq_mmt_reduced()\r\n", + "bi_co2e" + ], "outputs": [ { + "output_type": "execute_result", "data": { + "text/plain": [ + " World OECD90 Eastern Europe Asia (Sans Japan) Middle East and Africa Latin America China India EU USA\n", + "Year \n", + "2014 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2015 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2016 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2017 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2018 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2019 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2020 78.521223 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2021 89.731723 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2022 100.641071 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2023 111.255350 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2024 121.580472 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2025 131.624606 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2026 151.200284 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2027 157.831089 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2028 164.590307 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2029 171.460410 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2030 179.145789 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2031 185.463192 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2032 192.560849 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2033 199.699347 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2034 206.861193 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2035 212.905981 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2036 221.184782 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2037 228.311607 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2038 235.391842 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2039 242.407851 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2040 250.928085 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2041 256.178047 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2042 262.897327 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2043 269.482775 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2044 275.916966 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2045 281.215273 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2046 288.261917 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2047 294.137871 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2048 299.792955 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2049 305.217274 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2050 310.606903 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2051 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2052 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2053 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2054 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2055 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2056 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2057 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2058 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2059 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0\n", + "2060 0.000000 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0" + ], "text/html": [ "
\n", "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
WorldOECD90Eastern EuropeAsia (Sans Japan)Middle East and AfricaLatin AmericaChinaIndiaEUUSA
Year
20140.0000000.00.00.00.00.00.00.00.00.0
20150.0000000.00.00.00.00.00.00.00.00.0
20160.0000000.00.00.00.00.00.00.00.00.0
20170.0000000.00.00.00.00.00.00.00.00.0
20180.0000000.00.00.00.00.00.00.00.00.0
20190.0000000.00.00.00.00.00.00.00.00.0
202039.4127450.00.00.00.00.00.00.00.00.0
202158.5877960.00.00.00.00.00.00.00.00.0
202277.4155730.00.00.00.00.00.00.00.00.0
202395.9036120.00.00.00.00.00.00.00.00.0
2024114.0617640.00.00.00.00.00.00.00.00.0
2025131.8989990.00.00.00.00.00.00.00.00.0
2026149.4209150.00.00.00.00.00.00.00.00.0
2027166.6318250.00.00.00.00.00.00.00.00.0
2028183.5383100.00.00.00.00.00.00.00.00.0
2029200.1471700.00.00.00.00.00.00.00.00.0
2030216.4650080.00.00.00.00.00.00.00.00.0
2031232.4977560.00.00.00.00.00.00.00.00.0
2032248.1902180.00.00.00.00.00.00.00.00.0
2033263.5213590.00.00.00.00.00.00.00.00.0
2034278.5016460.00.00.00.00.00.00.00.00.0
2035293.1410490.00.00.00.00.00.00.00.00.0
2036307.4488300.00.00.00.00.00.00.00.00.0
2037321.4337440.00.00.00.00.00.00.00.00.0
2038335.1043830.00.00.00.00.00.00.00.00.0
2039348.4689880.00.00.00.00.00.00.00.00.0
2040361.5354160.00.00.00.00.00.00.00.00.0
2041374.3113880.00.00.00.00.00.00.00.00.0
2042386.8040210.00.00.00.00.00.00.00.00.0
2043399.0195730.00.00.00.00.00.00.00.00.0
2044410.9637560.00.00.00.00.00.00.00.00.0
2045422.6422380.00.00.00.00.00.00.00.00.0
2046434.0611710.00.00.00.00.00.00.00.00.0
2047445.2265880.00.00.00.00.00.00.00.00.0
2048456.1437100.00.00.00.00.00.00.00.00.0
2049466.8174520.00.00.00.00.00.00.00.00.0
2050477.2527600.00.00.00.00.00.00.00.00.0
20510.0000000.00.00.00.00.00.00.00.00.0
20520.0000000.00.00.00.00.00.00.00.00.0
20530.0000000.00.00.00.00.00.00.00.00.0
20540.0000000.00.00.00.00.00.00.00.00.0
20550.0000000.00.00.00.00.00.00.00.00.0
20560.0000000.00.00.00.00.00.00.00.00.0
20570.0000000.00.00.00.00.00.00.00.00.0
20580.0000000.00.00.00.00.00.00.00.00.0
20590.0000000.00.00.00.00.00.00.00.00.0
20600.0000000.00.00.00.00.00.00.00.00.0
\n", - "
" - ] - }, - "metadata": {}, - "execution_count": 28 - } - ], + "outputs": [], "metadata": { "scrolled": true } @@ -987,7 +193,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "source": [ "# Run the VB macros first!\r\n", "\r\n", @@ -1001,7 +207,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "source": [ "# Move the resulting file where it belongs.\r\n", "\r\n", @@ -1024,7 +230,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "source": [ "solution_name='residentialglass'\r\n", "solution_testfile_name=f\"test_{solution_name}.py\"\r\n", @@ -1049,119 +255,11 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "source": [ "!python -m pytest $outdir" ], - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "============================= test session starts =============================\n", - "platform win32 -- Python 3.9.6, pytest-6.2.4, py-1.10.0, pluggy-0.13.1\n", - "rootdir: C:\\Working\\solutions, configfile: tox.ini\n", - "collected 2 items\n", - "\n", - "solution\\composting\\tests\\test_composting.py .F [100%]\n", - "\n", - "================================== FAILURES ===================================\n", - "___________________________ test_composting_results ___________________________\n", - "\n", - "scenario_skip = None, test_skip = None, test_only = None\n", - "\n", - " @pytest.mark.slow\n", - " def test_composting_results(scenario_skip=None, test_skip=None, test_only=None):\n", - " \"\"\"Test computed results against stored Excel results\"\"\"\n", - " scenario_skip = scenario_skip or SCENARIO_SKIP\n", - " test_skip = test_skip or TEST_SKIP\n", - "> expected_result_tester.one_solution_tester(\n", - " solution_name,\n", - " expected_file, is_land=False,\n", - " scenario_skip=scenario_skip, test_skip=test_skip, test_only=test_only)\n", - "\n", - "solution\\composting\\tests\\test_composting.py:31: \n", - "_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _\n", - "tools\\expected_result_tester.py:1031: in one_solution_tester\n", - " check_excel_against_object(obj, zf, scenario_name, to_verify,\n", - "tools\\expected_result_tester.py:998: in check_excel_against_object\n", - " compare_dataframes(actual_df=actual_df, expected_df=expected_df,\n", - "_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _\n", - "\n", - "actual_df = Year World OECD90 Eastern Europe Asia (Sans Japan) Middle East and Africa Latin America Chin....559515 364.793698 2679.446828 2354.595021 804.193008 9.203801 84.523682 79.64090 45.54\n", - "expected_df = 16 17 18 19 20 21 22 23 24 25 ... 2060 14762.419566 8551.933590 364.776260 2643.830672 2358.811265 804.191614 4.6019 42.261841 39.82045 22.77\n", - "description = 'Solution: Composting Scenario: PDS-34p2050-May2020 Unit Adoption Calculations Q135:AA181'\n", - "mask = 16 17 18 19 20 21 22 23 24 25 26 Asia (Sans Japan) China EU Eas...alse False False False False False False False False\n", - "\n", - "[94 rows x 22 columns]\n", - "absignore = None\n", - "\n", - " def compare_dataframes(actual_df, expected_df, description='', mask=None, absignore=None):\n", - " \"\"\"Compare two dataframes and print where they differ.\"\"\"\n", - " nerrors = 0\n", - " if actual_df.shape != expected_df.shape:\n", - " raise AssertionError(description + '\\nDataFrames differ in shape: ' +\n", - " str(actual_df.shape) + \" versus \" + str(expected_df.shape))\n", - " (nrows, ncols) = actual_df.shape\n", - " msg = ''\n", - " rel = 1e-6 if absignore else None # if abs & !rel, rel is ignored. We want rel.\n", - " \n", - " \n", - " for r in range(nrows):\n", - " for c in range(ncols):\n", - " if mask is not None:\n", - " mask.iloc[r, c]\n", - " if mask is not None and mask.iloc[r, c]:\n", - " continue\n", - " matches = True\n", - " act = actual_df.iloc[r, c]\n", - " exp = expected_df.iloc[r, c]\n", - " if isinstance(act, str) and isinstance(exp, str):\n", - " matches = (act == exp)\n", - " elif (pd.isna(act) or act == '' or act is None or act == 0 or act == pytest.approx(0.0)\n", - " or exp == pytest.approx(0.0)):\n", - " matches = (pd.isna(exp) or exp == '' or exp is None or exp == 0 or\n", - " exp == pytest.approx(0.0, abs=1e-7))\n", - " elif np.isinf(act):\n", - " # Excel #DIV/0! turns into NaN.\n", - " matches = pd.isna(exp) or np.isinf(exp)\n", - " else:\n", - " matches = (act == pytest.approx(exp, rel=rel, abs=absignore))\n", - " if not matches:\n", - " msg += \"Err [\" + str(r) + \"][\" + str(c) + \"] : \" + \\\n", - " \"'\" + str(act) + \"' != '\" + str(exp) + \"'\\n\"\n", - " nerrors += 1\n", - " if nerrors > 10:\n", - " break\n", - " if msg:\n", - "> raise AssertionError(description + '\\nDataFrames differ:\\n' + msg)\n", - "E AssertionError: Solution: Composting Scenario: PDS-34p2050-May2020 Unit Adoption Calculations Q135:AA181\n", - "E DataFrames differ:\n", - "E Err [2][2] : '126.1526760593272' != '154.5267512826652'\n", - "E Err [2][3] : '5.77275458555166' != '5.755316301720526'\n", - "E Err [2][4] : '120.0576454813384' != '84.44148875649192'\n", - "E Err [2][5] : '10.63625345050044' != '14.852497777486196'\n", - "E Err [2][6] : '0.461675655780282' != '0.4602810302156386'\n", - "E Err [2][7] : '9.2038005152678' != '4.6019002576339'\n", - "E Err [2][8] : '84.5236824402514' != '42.2618412201257'\n", - "E Err [2][9] : '79.6409' != '39.82045'\n", - "E Err [2][10] : '45.54' != '22.77'\n", - "E Err [3][2] : '222.9054876615004' != '251.2795628848384'\n", - "E Err [3][3] : '8.905047272119651' != '8.887608988288516'\n", - "\n", - "tools\\expected_result_tester.py:954: AssertionError\n", - "---------------------------- Captured stdout call -----------------------------\n", - "Checking PDS-34p2050-May2020\n", - "TAM Data\n", - "Helper Tables\n", - "Emissions Factors\n", - "Unit Adoption Calculations\n", - "=========================== short test summary info ===========================\n", - "FAILED solution/composting/tests/test_composting.py::test_composting_results\n", - "========================= 1 failed, 1 passed in 5.57s =========================\n" - ] - } - ], + "outputs": [], "metadata": { "scrolled": true } @@ -1186,33 +284,12 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "source": [ "import solution.afforestation.tests.test_afforestation as mytests\r\n", "mytests.test_afforestation_results(scenario_skip=[1],test_only=['First Cost', 'Operating Cost'], test_skip=['C37:C82'])" ], - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Checking scenario 0: PDS-100p2050-Drawdown-CustomPDS-high-Aug2019\n", - "**** Skipped scenario 1 'PDS-100p2050-Drawdown-CustomPDS-High-July2019'\n", - "Checking scenario 2: PDS-100p2050-Optimum-CustomPDS-HighgrowthKreidenweis-July2018\n", - "Checking scenario 3: PDS-100p2050-Optimum-PDScustom-high-kreidnweis-Aug2019\n", - "Checking scenario 4: PDS-57p2050-Plausible-cusomPDS-avg-30Jan2020\n", - "Checking scenario 5: PDS-57p2050-Plausible-CustomPDS-Avg-Jan2020\n", - "Checking scenario 6: PDS-62p2050-Plausible-PDSCustom-low-Nov2019\n", - "Checking scenario 7: PDS-65p2050-drawdown-customPDS-30Jan2020\n", - "Checking scenario 8: PDS-65p2050-Drawdown-CustomPDS-high0.5stdv-Jan2020\n", - "Checking scenario 9: PDS-69p2050-Drawdown-PDSCustom-avg-Nov2019\n", - "Checking scenario 10: PDS-82p2050-Optimum-PDSCustom-high-Nov2019\n", - "Checking scenario 11: PDS-84p2050-Plausible-PDScustom-low-BookVersion1\n", - "Checking scenario 12: PDS-87p2050-plausible-avg-aug\n", - "Checking scenario 13: PDS-99p2050-Drawdown-Optimum-PDScustom-high-BookVersion1\n" - ] - } - ], + "outputs": [], "metadata": {} }, { @@ -1238,11 +315,11 @@ " The second part of the problem is because Excel itself is relying on those missing values. This is a case where it is legitimate to exclude\r\n", " tests on a permanent basis. \r\n", " \r\n", - " * Sometimes Excel workbooks compute values on the Advanced Controls tab, and those computations override the Scenario values. In Excel, when\r\n", - " you load a scenario, it copies all the scenario information from the ScenarioRecord tab into the places it belongs on the Advanced Controls\r\n", - " tab. But some Excel workbooks have created additional computations to calculate those values, and the formula overrides the scenario value.\r\n", - " The correct thing to do here is to remove the offending formula from the Excel, and re-create the expected.zip file with it out of the way.\r\n", + " * Sometimes Excel workbooks compute values on the Advanced Controls tab, and those computations override the Scenario values. In Excel, when you load a scenario, it copies all the scenario information from the ScenarioRecord tab into the places it belongs on the Advanced Controls tab. But some Excel workbooks have created additional computations to calculate those values, and the formula overrides the scenario value. The correct thing to do here is to remove the offending formula from the Excel, and re-create the expected.zip file with it out of the way.\r\n", " If you do this, __be sure to document that in the changelog__.\r\n", + " * Sometimes, the names of data sets (particularly adoptions) get changed, and older scenarios are never updated to the new name.\r\n", + " You can fix this either by changing the name in the scenario to match the current data source name as found in `__init__.py`\r\n", + " (usually it should be obvious which scenario), or by deleting the scenario, if it isn't an important one.\r\n", "\r\n", "# Tips\r\n", "\r\n", @@ -1258,11 +335,6 @@ "\r\n", "Are you looking at an excel formula with five nested `IF(...` expressions? Try [https://www.excelformulabeautifier.com/](https://www.excelformulabeautifier.com/). You're welcome.\r\n", "\r\n", - "## Limit the number of scenarios\r\n", - "\r\n", - "We don't actually want a dozen scenarios for each solution. Ideally, we would have the \"Book Edition 1\" scenarios (if the solution has them), and the most recent set of scenarios that don't look like someone was experimenting. We always want scenarios in sets of (PDS1, PDS2 and PDS3), which might also sometimes be labeled (Plausible, DrawDown and Optimum). To get rid of extra scenarios, just delete the scenario files from the `ac` directory (and document that you did so in changelog).\r\n", - "\r\n", - "Removing excess scenarios makes your testing go faster.\r\n", "\r\n", "## Look through changelog files for other solutions\r\n", "\r\n", @@ -1282,13 +354,6 @@ "Thank you for helping!" ], "metadata": {} - }, - { - "cell_type": "code", - "execution_count": null, - "source": [], - "outputs": [], - "metadata": {} } ], "metadata": { diff --git a/tools/Extraction_Guide.md b/tools/Extraction_Guide.md index 25b82c0db..37d3d6450 100644 --- a/tools/Extraction_Guide.md +++ b/tools/Extraction_Guide.md @@ -20,12 +20,13 @@ what the analysis is doing. You just need a bit of a holistic grasp of the over ## The Extraction Process -The Jupyter Notebook `Extraction_Guide.ipynb` walks you through steps 4-7 below. +The Jupyter Notebook `Extraction_Guide.ipynb` walks you through steps 4-9 below. 1. Obtain one of the Project Drawdown workbooks from PD, via hackathon, etc. 2. Prepare the workbook for extraction following the instructions below. 3. Set up a development environment following the instructions in `README.md` 4. Run the extraction step defined in `tools/solution_xls_extract.py` + 5. Clean up the generated scenarios. See the guidance in the Jupyter Notebook for more information. 5. Verify that the resulting solution code can be loaded and run 6. Create the `expected.zip` test file following the steps described in the Jupyter notebook and in `tools/CREATE_EXPECTED_ZIP.md` 7. Generate a new solution test file with the function `output_solution_test_file` in `tools/solution_xls_extract.py` @@ -33,7 +34,7 @@ The Jupyter Notebook `Extraction_Guide.ipynb` walks you through steps 4-7 below. 9. Document any modifications you had to make to solution code in a `changelog` file in the solution directory 10. Create a PR (Pull Request) for the results. -Note: it is _not_ reqired that the tests in step (8) run clean in order to PR the results. Some bugs that we are addressing are +Note: it is _not_ reqired that the tests in step (9) run clean in order to PR the results. Some bugs that we are addressing are systematic, and it makes more sense to upload the extracted model so they can be considered together. You should, however, complete all the steps in the list. @@ -41,7 +42,7 @@ You should, however, complete all the steps in the list. Make sure that you can open and run the Excel workbook: on the `Basic Controls` tab, find the 'Scenario to Customize' dropdown at cell C5. Select one of the scenarios from the dropdown (scroll up if the list appears empty), and then click the 'Load Scenario Inputs' button. You should see numbers appear in the other fields, and a graph appear to the right. -Several changes need to be made to the workbook itself to enable other extraction processes to work correctly. Follow instructions 1-7 in [this document](https://docs.google.com/document/d/1OiKg3_OOGjYOUdnHTQuZggsko5n31qv_YV4h77E3LHk/edit?usp=sharing). +Several changes may need to be made to the workbook itself to enable other extraction processes to work correctly. Follow instructions 1-7 in [this document](https://docs.google.com/document/d/1OiKg3_OOGjYOUdnHTQuZggsko5n31qv_YV4h77E3LHk/edit?usp=sharing). ## Handling Issues that Come Up @@ -52,19 +53,14 @@ For Excel Workbooks that follow the standard Project Drawdown templates, extract * Issues or variations in the Excel workbook itself Identifying and fixing these issues is part of the extraction process. When doing this, please note the following: - 1. **Do not make changes to the Excel workbooks without signoff from someone from Project Drawdown.** Even if you are convinced the Excel is incorrect. Do document your findings (open a github issue). - We need "official signoff" from Project Drawdown both to verify accuracy and to make sure that the master copies of the workbooks are also upated. - Instead of changing the Excel, consider solution-specific hacks to the extraction code (see next item). - 2. **Do make changes to the generated __init__.py file**. If there is a flaw in the generated code that makes it not match the Excel results, your first course of action should be to fix it in the `__init__.py` file. - 3. **Do make changes to the extraction code.** It is also acceptable to put solution-specific code into `solution_xls_extract.py` or related files. - Even hacky work-arounds are accepted here, since once extraction is complete, we will be done with it. - 4. **Cautiously make changes to the model code.** If you can find and fix bugs in code in the `/model`, `/tools` or `/data` directories, that is fantastic. These fixes should stand the test of time, however, not be hacks. - 5. **Changes to the test code accepted, but not required.** If there are problems that you are certain _are in the tests_, skipping those tests is acceptable. In this case, please open an issue describing the failing test, and why you believe the test is at fault. - -In all cases if you make customizations to *any* code, be sure to add comments inline next to the changes with your name, a date, and the -reason for the change. Include all changed code in your pull request. -We will use this information to determine which changes get into the code and which become issues that require further -investigation and fixes. + 1. **Do not make changes to the Excel workbooks without signoff from someone from Project Drawdown.** Even if you are convinced the Excel is incorrect. Do document your findings (open a github issue). We need "official signoff" from Project Drawdown both to verify accuracy and to make sure that the master copies of the workbooks are also upated. + Instead of changing the Excel, consider solution-specific hacks to the generated code (see next item). + 2. **Do make changes to the generated __init__.py file, or to scenario files**. If there is a flaw in the generated code that makes it not match the Excel results, your first course of action should be to fix it in the `__init__.py` file or other generated files. There are hints about things to check for in the accompanying Jupyter Notebook. + 3. **Cautiously make changes to the model code.** If you can find and fix bugs in code in the `/model`, `/tools` or `/data` directories, that is fantastic. These fixes should stand the test of time, however, not be hacks. + 4. **Changes to the test code accepted, but not required.** If there are problems that you are certain _are in the tests_, either fixing or skipping those tests is acceptable. If skipping, please open an issue describing the failing test, and why you believe the test is at fault. + 5. **Changes to the extraction code accepted, but not required**. If the extraction code can be modified to handle a situation better, and the change will work for all future models, then we welcome the contribution. But often it will be easier (and safer) to just fix the generated code directly. + +In all cases if you make customizations to *any* code, be sure to add comments inline next to the changes with your name, a date, and the reason for the change. Include all changed code in your pull request. Here's an example: ```python diff --git a/tools/solution_test_template.py b/tools/solution_test_template.py index 89343906f..1095f6627 100644 --- a/tools/solution_test_template.py +++ b/tools/solution_test_template.py @@ -17,11 +17,11 @@ TEST_SKIP = None def test_SOLUTION_loader(): - """Test that the solution can load a single scenario""" - (constructor,scenarios) = factory.one_solution_scenarios(solution_name) - assert len(scenarios) > 0 - ascenario = constructor(scenarios[0]) - assert ascenario is not None + """Test that the solution can load the defined scenarios""" + pds1 = factory.solution_pds_type(solution_name,"PDS1") + pds2 = factory.solution_pds_type(solution_name,"PDS2") + pds3 = factory.solution_pds_type(solution_name, "PDS3") + assert pds1 and pds2 and pds3 @pytest.mark.slow def test_SOLUTION_results(scenario_skip=None, test_skip=None, test_only=None): diff --git a/tools/solution_xls_extract.py b/tools/solution_xls_extract.py index 4ae6ef0a0..6537beba1 100755 --- a/tools/solution_xls_extract.py +++ b/tools/solution_xls_extract.py @@ -1705,7 +1705,13 @@ def output_solution_python_file(outputdir, xl_filename): write_scenario(filename=fname, s=s) f.write("scenarios = ac.load_scenarios_from_json(" "directory=THISDIR.joinpath('ac'), vmas=VMAs)\n") - f.write("\n\n") + f.write("\n") + + f.write('# These are the "default" scenarios to use for each of the drawdown categories.\n') + f.write('# They should be set to the most recent "official" set"\n') + f.write('PDS1 = "NOT SET"\n') + f.write('PDS2 = "NOT SET"\n') + f.write('PDS3 = "NOT SET"\n\n') f.write("class Scenario(scenario.Scenario):\n") f.write(" name = name\n")