diff --git a/docs/conf.py b/docs/conf.py index 4717e368..1d05a53d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,43 +12,44 @@ # import os import sys -sys.path.insert(0, os.path.abspath('../')) -sys.path.insert(0, os.path.abspath('../scripts')) -sys.path.insert(0, os.path.abspath('../oemof_b3/config/')) + +sys.path.insert(0, os.path.abspath("../")) +sys.path.insert(0, os.path.abspath("../scripts")) +sys.path.insert(0, os.path.abspath("../oemof_b3/config/")) # -- Project information ----------------------------------------------------- -project = 'oemof-B3' -copyright = '2020, Reiner Lemoine Institut' -author = 'Reiner Lemoine Institut' +project = "oemof-B3" +copyright = "2020, Reiner Lemoine Institut" +author = "Reiner Lemoine Institut" # The full version, including alpha/beta/rc tags -release = '0.0.5dev' +release = "0.0.5dev" # -- General configuration --------------------------------------------------- -master_doc = 'index' +master_doc = "index" # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.coverage', - 'sphinx.ext.napoleon', - 'sphinxcontrib.bibtex', - 'sphinx.ext.autosectionlabel' + "sphinx.ext.autodoc", + "sphinx.ext.coverage", + "sphinx.ext.napoleon", + "sphinxcontrib.bibtex", + "sphinx.ext.autosectionlabel", ] # specify bibfiles for sphinxcontrib.bibtex -bibtex_bibfiles = ['bibliography.bib'] +bibtex_bibfiles = ["bibliography.bib"] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # -- Options for HTML output ------------------------------------------------- @@ -65,7 +66,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # -- Options for Sphinx autodoc ---------------------------------------------- diff --git a/oemof_b3/tools/testing_pipeline.py b/oemof_b3/tools/testing_pipeline.py index c4d2d976..b5ba682b 100644 --- a/oemof_b3/tools/testing_pipeline.py +++ b/oemof_b3/tools/testing_pipeline.py @@ -1,7 +1,12 @@ +""" +This script contains functions to test the files and folders created +through the snakemake pipeline. +""" import os import subprocess import snakemake import shutil +import logging def install_with_extra(extra): @@ -147,6 +152,19 @@ def remove_raw_data_created(exists): def remove_test_data(path): + """ + This function removes test data. + + Inputs + ------- + path : str + Path of test data + + Outputs + ------- + None + + """ if os.path.isfile(path): os.remove(path) @@ -163,83 +181,169 @@ def get_abs_path_list(output_rule_list): Outputs ------- - absolute_path_list : str - Absolute file path + absolute_path_list : list + Absolute file path in list """ - # Loop over each rule which is tested in the snakemake pipeline - for sublist in output_rule_list: - # Get absolute path - absolute_path_list = [os.path.join(os.getcwd(), entry) for entry in sublist] + # Get absolute path of rule + absolute_path_list = [os.path.abspath(entry) for entry in output_rule_list] return absolute_path_list +def file_name_extension(raw_file_path): + """ + This function rearranges the current absolute file path + with the new extension suffix '_original'. + + Inputs + ------- + raw_file_path : str + Absolute path of rule + + Outputs + ------- + renamed_path : str + + """ + # Get file extension + file_extension = raw_file_path[raw_file_path.rfind(".") + 1 :] + # Rename existing user data + renamed_file = rename_path( + raw_file_path, "." + file_extension, "_original." + file_extension + ) + + return renamed_file + + +def rule_test(sublist): + """ + This function runs the rule from the output rule sublist. + + Inputs + ------- + sublist : str + Path of rule + + Outputs + ------- + None + + """ + # Run the snakemake rule in this loop + output = snakemake.snakemake( + targets=sublist, + snakefile="Snakefile", + ) + + # Check if snakemake rule exited without error (true) + assert output, f"Snakemake rule failed for targets: {sublist}" + + # Log the success + logging.info(f"Snakemake rule executed successfully for targets: {sublist}") + + +def clean_file(sublist, delete_switch, renamed_path): + """ + This function removes test data files and reverts renamed files. + + Inputs + ------- + sublist : list of str + List of target paths of rules + delete_switch : bool + If True, delete the data created during the test run. + If False, do not delete the data. + renamed_path : list of str + List of renamed target paths + + Outputs + ------- + None + + """ + # Remove the file created for this test + for raw_file_path in sublist: + if os.path.exists(raw_file_path): + if delete_switch or renamed_path: + remove_test_data(raw_file_path) + + # If file had to be renamed revert the changes + for renamed_file in renamed_path: + if os.path.isfile(renamed_file): + file_extension = renamed_file[renamed_file.rfind(".") + 1 :] + rename_path( + renamed_file, + "_original." + file_extension, + "." + file_extension, + ) + + def pipeline_file_output_test(delete_switch, output_rule_list): + """ + This function tests the Snakemake pipeline for a list of output rules + and reverts all changes made in the target directory. + + Inputs + ------- + delete_switch : bool + If True, delete the data created during the test run. + If False, do not delete the data. + output_rule_list : list of str + Nested list with sublist containing paths to target files + associated with a specific rule. + + Outputs + ------- + None + + """ # Raw data is needed for some rules and therefore is created if missing raw_data_exists = check_raw_data_exists() # Loop over each rule which is tested in the snakemake pipeline for sublist in output_rule_list: - # Get absolute path - absolute_path_list = [os.path.join(os.getcwd(), entry) for entry in sublist] + # Get absolute path of sublist + absolute_path_list = get_abs_path_list(sublist) renamed_path = [] for raw_file_path in absolute_path_list: + try: + # Check if file already exists in directory + if os.path.isfile(raw_file_path): + # Rename file with extension original + renamed_file = file_name_extension(raw_file_path) + renamed_path.append(renamed_file) + else: + # Check for the file with the _original suffix + original_file_path = raw_file_path.replace( + os.path.splitext(raw_file_path)[1], + "_original" + os.path.splitext(raw_file_path)[1], + ) + if os.path.exists(original_file_path): + raise FileExistsError( + f"File {original_file_path} already exists." + f"Please rename the file {raw_file_path} first." + ) - if os.path.isfile(raw_file_path): - # Get file extension - file_extension = raw_file_path[raw_file_path.rfind(".") + 1 :] - # Rename existing user data - renamed_file = rename_path( - raw_file_path, "." + file_extension, "_original." + file_extension - ) - renamed_path.append(renamed_file) + except FileNotFoundError as e: + print(e) + continue try: - # Run the snakemake rule in this loop - output = snakemake.snakemake( - targets=sublist, - snakefile="Snakefile", - ) - - # Check if snakemake rule exited without error (true) - assert output + # Run the snakemake rule + rule_test(sublist) # Check if the output file was created for raw_file_path in absolute_path_list: assert os.path.exists(raw_file_path) - for raw_file_path in sublist: - # Remove the file created for this test - if os.path.exists(raw_file_path): - if delete_switch or renamed_path: - remove_test_data(raw_file_path) - - # If file had to be renamed revert the changes - for renamed_file in renamed_path: - if os.path.isfile(renamed_file): - rename_path( - renamed_file, - "_original." + file_extension, - "." + file_extension, - ) + # Revert file changes + clean_file(sublist, delete_switch, renamed_path) except BaseException: - # Revert changes - for remove_raw_file_path in sublist: - # Remove the file created for this test - if os.path.exists(remove_raw_file_path): - remove_test_data(remove_raw_file_path) - - # If file had to be renamed revert the changes - for renamed_file in renamed_path: - if os.path.isfile(renamed_file): - rename_path( - renamed_file, - "_original." + file_extension, - "." + file_extension, - ) + # Revert file changes + clean_file(sublist, delete_switch, renamed_path) raise AssertionError( f"The workflow {raw_file_path} could not be executed correctly. " diff --git a/tests/test_pipeline_raw_data.py b/tests/test_pipeline_raw_data.py index d6519274..e2d051d2 100644 --- a/tests/test_pipeline_raw_data.py +++ b/tests/test_pipeline_raw_data.py @@ -1,3 +1,7 @@ +""" +This script checks the snakemake pipeline for target rules that create +empty time series and scalars in the folder raw. +""" import os import snakemake import shutil diff --git a/tests/test_pipeline_resources_tables.py b/tests/test_pipeline_resources_tables.py index 07e041d0..df5ba9e1 100644 --- a/tests/test_pipeline_resources_tables.py +++ b/tests/test_pipeline_resources_tables.py @@ -1,3 +1,7 @@ +""" +This script checks the snakemake pipeline for target rules in folder _resources +and _tables in the results' folder. +""" import os from oemof_b3.tools.testing_pipeline import ( get_repo_path,