From 0c38d215e7cb2dba65012ed197ae64298df90bdc Mon Sep 17 00:00:00 2001 From: Kristina Thieme Date: Thu, 5 Sep 2024 16:51:04 +0200 Subject: [PATCH 1/6] functional, but needs polishing/refactoring --- README.md | 4 +- deepcave/layouts/general.py | 61 +- .../hyperparameter/configuration_cube 2.py | 440 +++ .../hyperparameter/parallel_coordinates 2.py | 431 +++ deepcave/runs/converters/optuna 2.py | 266 ++ docs/plugins/symbolic_explanations 2.rst | 22 + examples/api/importances 2.py | 36 + logs/README 2.rst | 11 + .../run_1/configspace 2.json | 1 + .../run_1/intensifier 2.json | 145 + .../run_1/optimization 2.json | 6 + .../run_1/runhistory 2.json | 2393 +++++++++++++++++ .../run_1/scenario 2.json | 155 ++ .../run_2/configspace 2.json | 1 + .../run_2/intensifier 2.json | 74 + .../run_2/optimization 2.json | 6 + .../run_2/runhistory 2.json | 766 ++++++ .../run_2/scenario 2.json | 155 ++ tests/test_runs/__init__ 2.py | 0 tests/test_utils/__init__ 2.py | 0 20 files changed, 4953 insertions(+), 20 deletions(-) create mode 100644 deepcave/plugins/hyperparameter/configuration_cube 2.py create mode 100644 deepcave/plugins/hyperparameter/parallel_coordinates 2.py create mode 100644 deepcave/runs/converters/optuna 2.py create mode 100644 docs/plugins/symbolic_explanations 2.rst create mode 100644 examples/api/importances 2.py create mode 100644 logs/README 2.rst create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/configspace 2.json create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/intensifier 2.json create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/optimization 2.json create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/runhistory 2.json create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/scenario 2.json create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/configspace 2.json create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/intensifier 2.json create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/optimization 2.json create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/runhistory 2.json create mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/scenario 2.json create mode 100644 tests/test_runs/__init__ 2.py create mode 100644 tests/test_utils/__init__ 2.py diff --git a/README.md b/README.md index 3354fcc5..41493d6f 100644 --- a/README.md +++ b/README.md @@ -69,8 +69,8 @@ DeepCAVE comes with some pre-evaluated runs to get a feeling for what DeepCAVE c If you cloned the repository from GitHub via `git clone https://github.com/automl/DeepCAVE.git`, you can try out some examples by exploring the `logs` directory inside the DeepCAVE dashboard. -For example, if you navigate to `logs/DeepCAVE`, you can view the run `mnist_pytorch` if you hit -the `+` button left to it. +For example, if you navigate to `logs/DeepCAVE/mnist_pytorch`, you can view its runs if you hit +the `+` button left to them. ## Features diff --git a/deepcave/layouts/general.py b/deepcave/layouts/general.py index 20c03da9..4f4dd755 100644 --- a/deepcave/layouts/general.py +++ b/deepcave/layouts/general.py @@ -142,8 +142,9 @@ def callback(run_paths: List[str]): # Add text to go to parent directory new_element = html.Div( [ - dbc.Button( - "+", id={"type": "general-dynamic-add-run", "index": -1}, disabled=True + html.I( + className="fas fa-folder-open fa-lg", + id={"type": "general-dynamic-add-run", "index": -1}, ), dbc.Button( "..", @@ -161,23 +162,47 @@ def callback(run_paths: List[str]): for i, run_path in enumerate(run_paths): run_name = run_handler.get_run_name(run_path) - new_element = html.Div( - [ - dbc.Button("+", id={"type": "general-dynamic-add-run", "index": i}), - dbc.Button( - run_name, - id={"type": "general-dynamic-change-directory", "index": i}, - color="link", - ), - dcc.Store( - id={"type": "general-dynamic-available-run-path", "index": i}, - data=run_path, - ), - ], - className="mb-1", - ) - children.append(new_element) + is_run = run_handler.add_run(run_path) + # Differenciate between run and directory for visibility reasons + if is_run: + run_handler.remove_run(run_path) + new_element = html.Div( + [ + dbc.Button( + "+", id={"type": "general-dynamic-add-run", "index": i}, size="sm" + ), + dbc.Button( + run_name, + id={"type": "general-dynamic-change-directory", "index": i}, + color="light", + disabled=True, + ), + dcc.Store( + id={"type": "general-dynamic-available-run-path", "index": i}, + data=run_path, + ), + ], + className="mb-1", + ) + else: + new_element = html.Div( + [ + html.I(className="fas fa-folder fa-lg"), + dbc.Button( + run_name, + id={"type": "general-dynamic-change-directory", "index": i}, + color="link", + ), + dcc.Store( + id={"type": "general-dynamic-available-run-path", "index": i}, + data=run_path, + ), + ], + className="mb-1", + ) + print(new_element) + children.append(new_element) if len(children) == 0: return html.Div("No runs found.") diff --git a/deepcave/plugins/hyperparameter/configuration_cube 2.py b/deepcave/plugins/hyperparameter/configuration_cube 2.py new file mode 100644 index 00000000..ee6a4051 --- /dev/null +++ b/deepcave/plugins/hyperparameter/configuration_cube 2.py @@ -0,0 +1,440 @@ +# noqa: D400 +""" +# ConfigurationCube + +This module provides utilities for visualizing and creating a configuration cube. + +The configuration cube displays configurations and their score on the objective. + +## Classes + - ConfigurationCube: A plugin for visualizing a configuration cube. +""" + +from typing import Any, Callable, Dict, List, Tuple + +import dash_bootstrap_components as dbc +import numpy as np +import pandas as pd +import plotly.graph_objs as go +from dash import dcc, html +from dash.exceptions import PreventUpdate + +from deepcave import config +from deepcave.plugins.dynamic import DynamicPlugin +from deepcave.runs import AbstractRun, Status +from deepcave.utils.compression import deserialize, serialize +from deepcave.utils.layout import ( + get_checklist_options, + get_select_options, + get_slider_marks, + help_button, +) +from deepcave.utils.logs import get_logger +from deepcave.utils.styled_plotty import ( + get_hovertext_from_config, + get_hyperparameter_ticks, + save_image, +) + +logger = get_logger(__name__) + + +class ConfigurationCube(DynamicPlugin): + """A plugin for visualizing a configuration cube.""" + + id = "ccube" + name = "Configuration Cube" + icon = "fas fa-cube" + activate_run_selection = True + help = "docs/plugins/configuration_cube.rst" + + @staticmethod + def get_input_layout(register: Callable) -> List[dbc.Row]: + """ + Get the layout for the input block. + + Parameters + ---------- + register : Callable + Method to register (user) variables. + The register_input function is located in the Plugin superclass. + + Returns + ------- + List[dbc.Row] + Layouts for the input block. + """ + return [ + dbc.Row( + [ + dbc.Col( + [ + dbc.Label("Objective"), + dbc.Select( + id=register("objective_id", ["value", "options"], type=int), + placeholder="Select objective ...", + ), + ], + md=6, + ), + dbc.Col( + [ + dbc.Label("Budget"), + help_button( + "Budget refers to the multi-fidelity budget. " + "Combined budget means that the trial on the highest" + " evaluated budget is used. \n " + "Note: Selecting combined budget might be misleading if" + " a time objective is used. Often, higher budget take " + " longer to evaluate, which might negatively influence " + " the results." + ), + dbc.Select( + id=register("budget_id", ["value", "options"], type=int), + placeholder="Select budget ...", + ), + ], + md=6, + ), + ], + ), + ] + + @staticmethod + def get_filter_layout(register: Callable) -> List[html.Div]: + """ + Get the layout for the filter block. + + Parameters + ---------- + register : Callable + Method to register (user) variables. + The register_input function is located in the Plugin superclass. + + Returns + ------- + List[html.Div] + Layouts for the filter block. + """ + return [ + html.Div( + [ + dbc.Label("Number of Configurations"), + help_button( + "The number of configurations to show, these are ordered based on" + " the time at which they were evaluated." + ), + dcc.Slider( + id=register("n_configs", ["value", "min", "max", "marks"]), step=None + ), + ], + className="mb-3", + ), + html.Div( + [ + dbc.Label("Hyperparameters"), + help_button("Which hyperparameters to show. Maxium 3 active."), + dbc.Checklist( + id=register("hyperparameter_names", ["value", "options"]), inline=True + ), + ] + ), + ] + + def load_inputs(self) -> Dict[str, Any]: + """ + Load the content for the defined inputs in 'get_input_layout' and 'get_filter_layout'. + + This method is necessary to pre-load contents for the inputs. + So, if the plugin is called for the first time or there are no results in the cache, + the plugin gets its content from this method. + + Returns + ------- + Dict[str, Any] + The content to be filled. + """ + return { + "n_configs": {"min": 0, "max": 0, "marks": get_slider_marks(), "value": 0}, + "hyperparameter_names": {"options": get_checklist_options(), "value": []}, + } + + def load_dependency_inputs(self, run, _, inputs) -> Dict[str, Any]: # type: ignore + """ + Work like 'load_inputs' but called after inputs have changed. + + It is restricted to three Hyperparameters. + + Note + ---- + Only the changes have to be returned. + The returned dictionary will be merged with the inputs. + + Parameters + ---------- + run + The selected run. + inputs + Current content of the inputs. + + Returns + ------- + Dict[str, Any] + The dictionary with the changes. + """ + # Prepare objectives + objective_names = run.get_objective_names() + objective_ids = run.get_objective_ids() + objective_options = get_select_options(objective_names, objective_ids) + objective_value = inputs["objective_id"]["value"] + + # Prepare budgets + budgets = run.get_budgets(human=True) + budget_ids = run.get_budget_ids() + budget_options = get_select_options(budgets, budget_ids) + budget_value = inputs["budget_id"]["value"] + + # Prepare others + hp_names = list(run.configspace.keys()) + + # Get selected values + n_configs_value = inputs["n_configs"]["value"] + + # Pre-set values + if objective_value is None: + objective_value = objective_ids[0] + if budget_value is None: + budget_value = budget_ids[-1] + else: + budget_value = inputs["budget_id"]["value"] + + budget = run.get_budget(budget_value) + configs = run.get_configs(budget=budget) + if n_configs_value == 0: + n_configs_value = len(configs) + else: + if n_configs_value > len(configs): + n_configs_value = len(configs) + + # Restrict to three hyperparameters + selected_hps = inputs["hyperparameter_names"]["value"] + n_selected = len(selected_hps) + if n_selected > 3: + del selected_hps[0] + + return { + "objective_id": { + "options": objective_options, + "value": objective_value, + }, + "budget_id": { + "options": budget_options, + "value": budget_value, + }, + "n_configs": { + "min": 0, + "max": len(configs), + "marks": get_slider_marks(list(range(0, len(configs) + 1))), + "value": n_configs_value, + }, + "hyperparameter_names": { + "options": get_select_options(hp_names), + "value": selected_hps, + }, + } + + @staticmethod + def process(run: AbstractRun, inputs: Dict[str, Any]) -> Dict[str, str]: + """ + Return raw data based on the run and input data. + + Warning + ------- + The returned data must be JSON serializable. + + Note + ---- + The passed inputs are cleaned and therefore differs compared to 'load_inputs' + or 'load_dependency_inputs'. + Please see '_clean_inputs' for more information. + + Parameters + ---------- + run : AbstractRun + The selected run. + inputs : Dict[str, Any] + The input data. + + Returns + ------- + Dict[str, str] + The serialized dictionary. + """ + budget = run.get_budget(inputs["budget_id"]) + objective = run.get_objective(inputs["objective_id"]) + + df = run.get_encoded_data( + objectives=objective, budget=budget, statuses=Status.SUCCESS, include_config_ids=True + ) + return {"df": serialize(df)} + + @staticmethod + def get_output_layout(register: Callable) -> Tuple[dcc.Graph,]: + """ + Get the layout for the output block. + + Parameters + ---------- + register : Callable + Method to register output. + The register_output function is located in the Plugin superclass. + + Returns + ------- + Tuple[dcc.Graph,] + Layout for the output block. + """ + return ( + dcc.Graph( + register("graph", "figure"), + style={"height": config.FIGURE_HEIGHT}, + config={"toImageButtonOptions": {"scale": config.FIGURE_DOWNLOAD_SCALE}}, + ), + ) + + @staticmethod + def load_outputs(run, inputs, outputs) -> go.Figure: # type: ignore + """ + Read in the raw data and prepares them for the layout. + + Note + ---- + The passed inputs are cleaned and therefore differs compared to 'load_inputs' + or 'load_dependency_inputs'. + Please see '_clean_inputs' for more information. + + Parameters + ---------- + run + The selected run. + inputs + Input and filter values from the user. + outputs + Raw output from the run. + + Returns + ------- + go.Figure + The output figure. + """ + df = deserialize(outputs["df"], dtype=pd.DataFrame) + hp_names = inputs["hyperparameter_names"] + n_configs = inputs["n_configs"] + objective_id = inputs["objective_id"] + objective = run.get_objective(objective_id) + budget = run.get_budget(inputs["budget_id"]) + df = df.groupby(df.columns.drop(objective.name).to_list(), as_index=False).mean() + df.index = df.index.astype("str") + + # Limit to n_configs + idx = [str(i) for i in range(n_configs, len(df))] + df = df.drop(idx) + + costs = df[objective.name].values.tolist() + config_ids = df["config_id"].values.tolist() + data = [] + + # Specify layout kwargs + layout_kwargs = {} + if n_configs > 0 and len(hp_names) > 0: + for i, (hp_name, axis_name) in enumerate(zip(hp_names, ["xaxis", "yaxis", "zaxis"])): + hp = run.configspace[hp_name] + values = df[hp_name].values.tolist() + + tickvals, ticktext = get_hyperparameter_ticks(hp, ticks=4, include_nan=True) + layout_kwargs[axis_name] = { + "tickvals": tickvals, + "ticktext": ticktext, + "title": hp_name, + } + data.append(values) + + if len(data) == 0: + raise PreventUpdate + + # Transforms data to values + values = np.transpose(np.array(data)).tolist() + + # Now the duplicates are filtered + filtered_data: Dict[Tuple[int, float], Tuple] = {} + for config_id, cost, v in zip(config_ids, costs, values): + v = tuple(v) # Make hashable + key = (config_id, cost) + + if v in filtered_data.values(): + old_key = list(filtered_data.keys())[list(filtered_data.values()).index(v)] + old_cost = old_key[1] + + if objective.optimize == "lower": + if old_cost < cost: + continue + + if objective.optimize == "upper": + if old_cost > cost: + continue + + # Otherwise it is replaced + del filtered_data[old_key] + + filtered_data[key] = v + + # Replace data + config_ids = [k[0] for k in filtered_data.keys()] + costs = [k[1] for k in filtered_data.keys()] + + # Specify scatter kwargs + scatter_kwargs = { + "mode": "markers", + "marker": { + "size": 5, + "color": costs, + "colorbar": {"thickness": 30, "title": objective.name}, + }, + "hovertext": [ + get_hovertext_from_config(run, config_id, budget) for config_id in config_ids + ], + "meta": {"colorbar": costs}, + "hoverinfo": "text", + } + + if len(data) >= 1: + x = [v[0] for v in filtered_data.values()] + if len(data) >= 2: + y = [v[1] for v in filtered_data.values()] + if len(data) >= 3: + z = [v[2] for v in filtered_data.values()] + + if len(data) == 3: + trace = go.Scatter3d(x=x, y=y, z=z, **scatter_kwargs) + layout = go.Layout( + {"scene": {**layout_kwargs}}, + font=dict(size=config.FIGURE_FONT_SIZE), + ) + else: + if len(data) == 1: + y = [0 for _ in x] + trace = go.Scatter(x=x, y=y, **scatter_kwargs) + elif len(data) == 2: + trace = go.Scatter(x=x, y=y, **scatter_kwargs) + else: + trace = go.Scatter(x=[], y=[]) + layout = go.Layout( + **layout_kwargs, + font=dict(size=config.FIGURE_FONT_SIZE), + ) + + figure = go.Figure(data=trace, layout=layout) + figure.update_layout(dict(margin=config.FIGURE_MARGIN)) + save_image(figure, "configuration_cube.pdf") + + return figure diff --git a/deepcave/plugins/hyperparameter/parallel_coordinates 2.py b/deepcave/plugins/hyperparameter/parallel_coordinates 2.py new file mode 100644 index 00000000..b2b0bb84 --- /dev/null +++ b/deepcave/plugins/hyperparameter/parallel_coordinates 2.py @@ -0,0 +1,431 @@ +# noqa: D400 + +""" +# ParallelCoordinates + +This module provides utilities for visualizing the parallel coordinates. + +## Classes + - ParallelCoordinates : Can be used for visualizing the parallel coordinates. +""" + +from typing import Any, Callable, Dict, List + +from collections import defaultdict + +import dash_bootstrap_components as dbc +import numpy as np +import pandas as pd +import plotly.graph_objs as go +from dash import dcc, html +from dash.exceptions import PreventUpdate + +from deepcave import config +from deepcave.constants import VALUE_RANGE +from deepcave.evaluators.fanova import fANOVA +from deepcave.plugins.static import StaticPlugin +from deepcave.utils.compression import deserialize, serialize +from deepcave.utils.layout import get_checklist_options, get_select_options, help_button +from deepcave.utils.logs import get_logger +from deepcave.utils.styled_plotty import get_hyperparameter_ticks, save_image + +logger = get_logger(__name__) + + +class ParallelCoordinates(StaticPlugin): + """Can be used for visualizing the parallel coordinates.""" + + id = "parallel_coordinates" + name = "Parallel Coordinates" + icon = "far fa-map" + activate_run_selection = True + help = "docs/plugins/parallel_coordinates.rst" + + @staticmethod + def get_input_layout(register: Callable) -> List[Any]: + """ + Get the layout for the input block. + + Parameters + ---------- + register : Callable + Method to regsiter (user) variables. + The register_input function is located in the Plugin superclass. + + Returns + ------- + List[Any] + The layouts for the input block. + """ + return [ + dbc.Row( + [ + dbc.Col( + [ + dbc.Label("Objective"), + dbc.Select( + id=register("objective_id", ["value", "options"], type=int), + placeholder="Select objective ...", + ), + ], + md=6, + ), + dbc.Col( + [ + dbc.Label("Budget"), + help_button( + "Budget refers to the multi-fidelity budget. " + "Combined budget means that the trial on the highest evaluated" + " budget is used. \n " + "Note: Selecting combined budget might be misleading if a time" + " objective is used. Often, higher budget take longer to evaluate," + " which might negatively influence the results." + ), + dbc.Select( + id=register("budget_id", ["value", "options"], type=int), + placeholder="Select budget ...", + ), + ], + md=6, + ), + ], + className="mb-3", + ), + html.Div( + [ + dbc.Label("Show Important Hyperparameters"), + help_button( + "Only the most important hyperparameters are shown which are " + "calculated by fANOVA using 10 trees. The more left a " + "hyperparameter stands, the more important it is. However, activating " + "this option might take longer." + ), + dbc.Select( + id=register("show_important_only", ["value", "options"]), + placeholder="Select ...", + ), + ] + ), + ] + + @staticmethod + def get_filter_layout(register: Callable) -> List[Any]: + """ + Get the layout for the filter block. + + Parameters + ---------- + register : Callable + Method to register (user) variables. + The register_input function is located in the Plugin superclass. + + Returns + ------- + List[Any] + The layouts for the filter block. + """ + return [ + dbc.Row( + [ + dbc.Col( + [ + dbc.Label("Limit Hyperparameters"), + help_button( + "Shows either the n most important hyperparameters (if show " + "important hyperparameters is true) or the first n selected " + "hyperparameters." + ), + dbc.Input(id=register("n_hps", "value"), type="number"), + ], + md=6, + ), + dbc.Col( + [ + dbc.Label("Show Unsuccessful Configurations"), + help_button("Whether to show all configurations or only failed ones."), + dbc.Select( + id=register("show_unsuccessful", ["value", "options"]), + placeholder="Select ...", + ), + ], + md=6, + ), + ], + ), + html.Div( + [ + dbc.Label("Hyperparameters"), + dbc.Checklist( + id=register("hyperparameter_names", ["value", "options"]), inline=True + ), + ], + className="mt-3", + id=register("hide_hps", ["hidden"]), + ), + ] + + def load_inputs(self) -> Dict[str, Dict[str, Any]]: + """ + Load the content for the defined inputs in 'get_input_layout' and 'get_filter_layout'. + + This method is necessary to pre-load contents for the inputs. + So, if the plugin is called for the first time or there are no results in the cache, + the plugin gets its content from this method. + + Returns + ------- + Dict[str, Dict[str, Any]] + Content to be filled. + """ + return { + "show_important_only": {"options": get_select_options(binary=True), "value": "true"}, + "show_unsuccessful": {"options": get_select_options(binary=True), "value": "false"}, + "n_hps": {"value": 0}, + "hyperparameter_names": {"options": get_checklist_options(), "value": []}, + "hide_hps": {"hidden": True}, + } + + def load_dependency_inputs(self, run, _, inputs) -> Dict[str, Any]: # type: ignore + """ + Work like 'load_inputs' but called after inputs have changed. + + Note + ---- + Only the changes have to be returned. + The returned dictionary will be merged with the inputs. + + Parameters + ---------- + run + The selected run. + inputs + Current content of the inputs. + + Returns + ------- + Dict[str, Any] + The dictionary with the changes. + """ + # Prepare objectives + objective_names = run.get_objective_names() + objective_ids = run.get_objective_ids() + objective_options = get_select_options(objective_names, objective_ids) + objective_value = inputs["objective_id"]["value"] + + # Prepare budgets + budgets = run.get_budgets(human=True) + budget_ids = run.get_budget_ids() + budget_options = get_checklist_options(budgets, budget_ids) + budget_value = inputs["budget_id"]["value"] + + # Prepare others + n_hps = inputs["n_hps"]["value"] + hp_names = list(run.configspace.keys()) + + if inputs["show_important_only"]["value"] == "true": + hp_options = [] + hp_value = inputs["hyperparameter_names"]["value"] + hidden = True + else: + hp_options = get_select_options(hp_names) + + values = inputs["hyperparameter_names"]["value"] + if len(values) == 0: + values = hp_names + + hp_value = values + hidden = False + + if objective_value is None: + objective_value = objective_ids[0] + budget_value = budget_ids[-1] + hp_value = hp_names + + if n_hps == 0: + n_hps = len(hp_names) + + return { + "objective_id": { + "options": objective_options, + "value": objective_value, + }, + "budget_id": { + "options": budget_options, + "value": budget_value, + }, + "hyperparameter_names": { + "options": hp_options, + "value": hp_value, + }, + "n_hps": {"value": n_hps}, + "hide_hps": {"hidden": hidden}, + } + + @staticmethod + def process(run, inputs) -> Dict[str, Any]: # type: ignore + """ + Return raw data based on a run and input data. + + Warning + ------- + The returned data must be JSON serializable. + + Note + ---- + The passed inputs are cleaned and therefore differs compared to 'load_inputs' + or 'load_dependency_inputs'. + Please see '_clean_inputs' for more information. + + Parameters + ---------- + run : AbstractRun + The run to process. + inputs : Dict[str, Any] + The input data. + + Returns + ------- + Dict[str, Any] + The serialized dictionary. + """ + budget = run.get_budget(inputs["budget_id"]) + objective = run.get_objective(inputs["objective_id"]) + df = run.get_encoded_data(objective, budget) + df = df.groupby(df.columns.drop(objective.name).to_list(), as_index=False).mean() + df = serialize(df) + result: Dict[str, Any] = {"df": df} + + if inputs["show_important_only"]: + # Let's run a quick fANOVA here + evaluator = fANOVA(run) + evaluator.calculate(objective, budget, n_trees=10, seed=0) + importances_dict = evaluator.get_importances() + importances = {u: v[0] for u, v in importances_dict.items()} + important_hp_names = sorted( + importances, key=lambda key: importances[key], reverse=False + ) + result["important_hp_names"] = important_hp_names + + return result + + @staticmethod + def get_output_layout(register: Callable) -> dcc.Graph: + """ + Get the layout for the output block. + + Parameters + ---------- + register : Callable + Method to register outputs. + The register_output function is located in the Plugin superclass. + + Returns + ------- + dcc.Graph + The layouts for the output block. + """ + return dcc.Graph( + register("graph", "figure"), + style={"height": config.FIGURE_HEIGHT}, + config={"toImageButtonOptions": {"scale": config.FIGURE_DOWNLOAD_SCALE}}, + ) + + @staticmethod + def load_outputs(run, inputs, outputs) -> go.Figure: # type: ignore + """ + Read in the raw data and prepare them for the layout. + + Note + ---- + The passed inputs are cleaned and therefore differs compared to 'load_inputs' + or 'load_dependency_inputs'. + Please see '_clean_inputs' for more information. + + Parameters + ---------- + run + The selected run. + inputs + The inputs and filter values fromt the user. + outputs + Raw output from the run. + + Returns + ------- + go.Figure + The output figure. + """ + objective = run.get_objective(inputs["objective_id"]) + objective_name = objective.name + + show_important_only = inputs["show_important_only"] + show_unsuccessful = inputs["show_unsuccessful"] + n_hps = inputs["n_hps"] + + if n_hps == "" or n_hps is None: + raise PreventUpdate + else: + n_hps = int(n_hps) + + if show_important_only: + hp_names = outputs["important_hp_names"] + else: + hp_names = inputs["hyperparameter_names"] + + hp_names = hp_names[:n_hps] + + df = outputs["df"] + df = deserialize(df, dtype=pd.DataFrame) + objective_values = [] + for value in df[objective_name].values: + b = np.isnan(value) + if not show_unsuccessful: + b = not b + if b: + objective_values += [value] + + data: defaultdict = defaultdict(dict) + for hp_name in hp_names: + values = [] + for hp_v, objective_v in zip(df[hp_name].values, df[objective_name].values): + b = np.isnan(objective_v) + if not show_unsuccessful: + b = not b + if b: + values += [hp_v] + + data[hp_name]["values"] = values + data[hp_name]["label"] = hp_name + data[hp_name]["range"] = VALUE_RANGE + + hp = run.configspace[hp_name] + tickvals, ticktext = get_hyperparameter_ticks(hp, ticks=4, include_nan=True) + + data[hp_name]["tickvals"] = tickvals + data[hp_name]["ticktext"] = ticktext + + if show_unsuccessful: + line = dict() + else: + data[objective_name]["values"] = objective_values + data[objective_name]["label"] = objective_name + line = dict( + color=data[objective_name]["values"], + showscale=True, + colorscale="aggrnyl", + ) + + figure = go.Figure( + data=go.Parcoords( + line=line, + dimensions=list([d for d in data.values()]), + labelangle=45, + ), + layout=dict( + margin=dict(t=150, b=50, l=100, r=0), + font=dict(size=config.FIGURE_FONT_SIZE), + ), + ) + save_image(figure, "parallel_coordinates.pdf") + + return figure diff --git a/deepcave/runs/converters/optuna 2.py b/deepcave/runs/converters/optuna 2.py new file mode 100644 index 00000000..a6cd1497 --- /dev/null +++ b/deepcave/runs/converters/optuna 2.py @@ -0,0 +1,266 @@ +# noqa: D400 +""" +# OptunaRun + +This module provides utilities to create an Optuna run. + +## Classes + - OptunaRun: Define an Optuna run object. +""" + +from typing import List, Union + +import pickle +from pathlib import Path + +from ConfigSpace import ( + Categorical, + Configuration, + ConfigurationSpace, + Float, + Integer, + Uniform, +) +from ConfigSpace.hyperparameters import Hyperparameter +from optuna.distributions import ( + CategoricalDistribution, + FloatDistribution, + IntDistribution, +) +from optuna.search_space import IntersectionSearchSpace +from optuna.study import StudyDirection +from optuna.trial import TrialState + +from deepcave.runs import Status +from deepcave.runs.objective import Objective +from deepcave.runs.run import Run +from deepcave.utils.hash import file_to_hash +from deepcave.utils.logs import get_logger + +logger = get_logger(__name__) + + +class OptunaRun(Run): + """ + Define an Optuna run object. + + Properties + ---------- + path : Path + The path to the run. + """ + + prefix = "Optuna" + _initial_order = 2 + + @staticmethod + def _get_pickle_file(path: Path) -> Path: + """ + Get the path to the pickle file from the directory path. + + Parameters + ---------- + path : Path + The path to the directory containing the pickle file. + + Returns + ------- + Path + The path to the pickle file. + """ + pickle_files = list(path.glob("*.pkl")) + if len(pickle_files) != 1: + raise RuntimeError("There should be exactly one pickle file in the directory.") + else: + return pickle_files[0] + + @property + def hash(self) -> str: + """ + Hash of the current run. + + If the hash changes, the cache has to be cleared. + This ensures that the cache always holds the latest results of the run. + + Returns + ------- + str + The hash of the run. + """ + if self.path is None: + return "" + + pickle_file = OptunaRun._get_pickle_file(self.path) + return file_to_hash(pickle_file) + + @property + def latest_change(self) -> Union[float, int]: + """ + Get the timestamp of the latest change. + + Returns + ------- + Union[float, int] + The latest change. + """ + if self.path is None: + return 0 + + pickle_file = OptunaRun._get_pickle_file(self.path) + return Path(pickle_file).stat().st_mtime + + @classmethod + def from_path(cls, path: Union[Path, str]) -> "OptunaRun": + """ + Based on working_dir/run_name/*, return a new trials object. + + Parameters + ---------- + path : Union[Path, str] + The path to base the trial object on. + + Returns + ------- + The Optuna run. + + Raises + ------ + RuntimeError + Instances are not supported. + """ + path = Path(path) + + # Load the optuna study from the file path + pickle_file_path = OptunaRun._get_pickle_file(path) + with open(pickle_file_path, "rb") as f: + optuna_study = pickle.load(f) + + # Read configspace + optuna_space = IntersectionSearchSpace(include_pruned=True).calculate(study=optuna_study) + configspace = ConfigurationSpace() + + hyperparameters: List[Hyperparameter] = [] + + for hp_name, hp in optuna_space.items(): + if isinstance(hp, FloatDistribution) or isinstance(hp, IntDistribution): + if hp.step is not None and hp.step != 1: + logger.warning( + f"Step is not supported. " + f'Step={hp.step} will be ignored for hyperparameter "{hp_name}".' + ) + + if isinstance(hp, FloatDistribution): + hyperparameters.append( + Float( + name=hp_name, + bounds=(hp.low, hp.high), + distribution=Uniform(), + default=None, + log=hp.log, + ) + ) + elif isinstance(hp, IntDistribution): + hyperparameters.append( + Integer( + name=hp_name, + bounds=(hp.low, hp.high), + distribution=Uniform(), + default=None, + log=hp.log, + ) + ) + elif isinstance(hp, CategoricalDistribution): + hyperparameters.append( + Categorical( + name=hp_name, + default=hp.choices[0], + items=hp.choices, + ) + ) + else: + raise ValueError( + ( + "The hyperparameters in the Optuna study must be of type " + "`FloatDistribution`, `IntDistribution` or `CategoricalDistribution`, " + f"but a hyperparameter of type {type(hp)} was given." + ) + ) + configspace.add(hyperparameters) + + n_objectives = max(len(trial.values) for trial in optuna_study.trials) + obj_list = list() + for i in range(n_objectives): + if optuna_study.metric_names is not None: + metric_name = optuna_study.metric_names[i] + else: + metric_name = f"Objective{i}" + optimize = "lower" if optuna_study.directions[i] == StudyDirection.MINIMIZE else "upper" + + obj_list.append( + Objective( + name=metric_name, + lower=None, + upper=None, + optimize=optimize, + ) + ) + + obj_list.append(Objective("Time")) + + # Let's create a new run object + run = OptunaRun(name=path.stem, configspace=configspace, objectives=obj_list, meta=None) + + # The path has to be set manually + run._path = path + + first_starttime = None + for trial in optuna_study.trials: + try: + config = Configuration(configspace, trial.params) + except ValueError: + raise ValueError( + f"Could not convert the configuration of trial {trial.number} to " + f"a ConfigSpace configuration.\nThis might be due to the " + f"configuration space containing conditionals or dynamic " + f"hyperparameter value ranges, which are currently not supported." + ) + + if first_starttime is None: + first_starttime = trial.datetime_start.timestamp() + + starttime = trial.datetime_start.timestamp() - first_starttime + endtime = trial.datetime_complete.timestamp() - first_starttime + + if trial.state == TrialState.COMPLETE: + status = Status.SUCCESS + elif trial.state == TrialState.FAIL: + status = Status.FAILED + elif trial.state == TrialState.PRUNED: + status = Status.PRUNED + elif trial.state == TrialState.RUNNING or trial["status"] == TrialState.WAITING: + continue + else: + status = Status.UNKNOWN + + cost = trial.values + + if status != Status.SUCCESS: + # Costs which failed, should not be included + cost = [None] * len(cost) if isinstance(cost, list) else None + time = None + else: + time = float(endtime - starttime) + + run.add( + costs=cost + [time] if isinstance(cost, list) else [cost, time], # type: ignore + config=config, + budget=0.0, + seed=-1, + start_time=starttime, + end_time=endtime, + status=status, + origin=None, + additional=None, + ) + + return run diff --git a/docs/plugins/symbolic_explanations 2.rst b/docs/plugins/symbolic_explanations 2.rst new file mode 100644 index 00000000..1b78e02b --- /dev/null +++ b/docs/plugins/symbolic_explanations 2.rst @@ -0,0 +1,22 @@ +Symbolic Explanations +==================== + +Symbolic Explanations allow to obtain explicit formulas quantifying the relation between +hyperparameter values and model performance by applying symbolic regression to meta-data collected +during hyperparameter optimization. + +The plugin is capable of answering similar questions as the Partial Dependencies plugin, i.e.: + +* How does the objective change with respect to one or two hyperparameters? For example, does the + accuracy increase if the learning rate decreases? +* Do multiple trials show similar behavior? + +While the Partial Dependencies plugin provides a plot describing the effects of hyperparameters on +the model performance, the Symbolic Explanations plugin additionally allows to obtain an explicit +formula capturing these effects. + +To learn more about Symbolic Explanations, please see the paper +`Symbolic Explanations for Hyperparameter Optimization +`_. + +.. image:: ../images/plugins/symbolic_explanations.png diff --git a/examples/api/importances 2.py b/examples/api/importances 2.py new file mode 100644 index 00000000..de4dad38 --- /dev/null +++ b/examples/api/importances 2.py @@ -0,0 +1,36 @@ +""" +Importances +^^^^^^^^^^^^^^^^^^^^ + +This example shows how to use the plugin Importances. +Note that other plugins use the same interfaces and can be used in the same fashion. +""" + +from deepcave.plugins.hyperparameter.importances import Importances +from deepcave.runs.converters.deepcave import DeepCAVERun +from pathlib import Path + + +if __name__ == "__main__": + # Instantiate the run + run = DeepCAVERun.from_path(Path("logs/DeepCAVE/minimal/run_2")) + + objective_id = run.get_objective_ids()[0] + budget_ids = run.get_budget_ids() + + # Instantiate the plugin + plugin = Importances() + inputs = plugin.generate_inputs( + hyperparameter_names=list(run.configspace.keys()), + objective_id=objective_id, + budget_ids=budget_ids, + method="global", + n_hps=3, + n_trees=10, + ) + # Note: Filter variables are not considered. + outputs = plugin.generate_outputs(run, inputs) + + # Finally, you can load the figure. Here, the filter variables play a role. + figure = plugin.load_outputs(run, inputs, outputs) + figure.write_image("examples/api/importances.png", scale=2.0) diff --git a/logs/README 2.rst b/logs/README 2.rst new file mode 100644 index 00000000..2cadd357 --- /dev/null +++ b/logs/README 2.rst @@ -0,0 +1,11 @@ +Example Logs +============ + +This directory contains exemplary run logs from hyperparameter optimization runs created with +AMLTK, BOHB, Optuna, and SMAC3. +Furthermore, it contains example logs created with the DeepCAVE recorder and example logs in +CSV format. + +After starting DeepCAVE, you can load these logs by navigating to the directory containing the run +log you would like to load on the `General` site and then selecting the run log via clicking on +the `+` next to it. \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/configspace 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/configspace 2.json new file mode 100644 index 00000000..be8aa231 --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_1/configspace 2.json @@ -0,0 +1 @@ +{"name": null, "hyperparameters": [{"type": "categorical", "name": "activation", "choices": ["logistic", "tanh", "relu"], "weights": null, "default_value": "tanh", "meta": null}, {"type": "uniform_int", "name": "n_layer", "lower": 1, "upper": 5, "default_value": 1, "log": false, "meta": null}, {"type": "uniform_int", "name": "n_neurons", "lower": 8, "upper": 256, "default_value": 10, "log": true, "meta": null}, {"type": "categorical", "name": "solver", "choices": ["lbfgs", "sgd", "adam"], "weights": null, "default_value": "adam", "meta": null}, {"type": "uniform_int", "name": "batch_size", "lower": 30, "upper": 300, "default_value": 200, "log": false, "meta": null}, {"type": "categorical", "name": "learning_rate", "choices": ["constant", "invscaling", "adaptive"], "weights": null, "default_value": "constant", "meta": null}, {"type": "uniform_float", "name": "learning_rate_init", "lower": 0.0001, "upper": 1.0, "default_value": 0.001, "log": true, "meta": null}], "conditions": [{"type": "IN", "child": "batch_size", "parent": "solver", "values": ["sgd", "adam"]}, {"type": "EQ", "child": "learning_rate", "parent": "solver", "value": "sgd"}, {"type": "IN", "child": "learning_rate_init", "parent": "solver", "values": ["sgd", "adam"]}], "forbiddens": [], "python_module_version": "1.2.0", "format_version": 0.4} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/intensifier 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/intensifier 2.json new file mode 100644 index 00000000..d7b218ec --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_1/intensifier 2.json @@ -0,0 +1,145 @@ +{ + "incumbent_ids": [ + 20 + ], + "rejected_config_ids": [ + 1, + 6, + 8, + 3, + 10, + 4, + 12, + 13, + 7, + 2, + 15, + 17, + 18, + 19, + 14, + 24, + 21, + 9, + 22, + 16, + 26, + 29, + 28, + 31, + 32, + 34, + 27, + 36, + 37, + 35, + 23, + 11, + 38, + 39, + 43, + 5, + 41, + 40, + 42, + 45, + 46, + 48, + 30, + 50, + 52, + 53, + 55, + 54, + 44, + 57, + 56, + 25, + 49 + ], + "incumbents_changed": 5, + "trajectory": [ + { + "config_ids": [ + 1 + ], + "costs": [ + 0.6699473847106159 + ], + "trial": 1, + "walltime": 0.4147176742553711 + }, + { + "config_ids": [ + 8 + ], + "costs": [ + 0.12855926957598263 + ], + "trial": 2, + "walltime": 0.7400827407836914 + }, + { + "config_ids": [ + 2 + ], + "costs": [ + 0.025597338285360594 + ], + "trial": 5, + "walltime": 1.215728759765625 + }, + { + "config_ids": [ + 14 + ], + "costs": [ + 0.02392757660167144 + ], + "trial": 13, + "walltime": 2.366025686264038 + }, + { + "config_ids": [ + 20 + ], + "costs": [ + 0.02115031466006397 + ], + "trial": 37, + "walltime": 8.94394326210022 + } + ], + "state": { + "queue": [ + [ + 33, + 4 + ], + [ + 51, + 4 + ], + [ + 58, + 2 + ], + [ + 47, + 4 + ], + [ + 59, + 2 + ], + [ + 60, + 2 + ], + [ + 61, + 2 + ] + ] + } +} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/optimization 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/optimization 2.json new file mode 100644 index 00000000..cd5ebb87 --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_1/optimization 2.json @@ -0,0 +1,6 @@ +{ + "used_walltime": 67.33127307891846, + "used_target_function_walltime": 494.0225176811218, + "last_update": 1712565076.753763, + "finished": true +} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/runhistory 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/runhistory 2.json new file mode 100644 index 00000000..5ab1ec0f --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_1/runhistory 2.json @@ -0,0 +1,2393 @@ +{ + "stats": { + "submitted": 154, + "finished": 154, + "running": 0 + }, + "data": [ + [ + 1, + null, + 209652396, + null, + 0.6699473847106159, + 0.32405996322631836, + 1, + 1712565009.4599066, + 1712565009.7847676, + {} + ], + [ + 2, + null, + 209652396, + null, + 0.025597338285360594, + 1.0813992023468018, + 1, + 1712565009.4976892, + 1712565010.5797143, + {} + ], + [ + 3, + null, + 209652396, + null, + 0.6583333333333333, + 1.1021058559417725, + 1, + 1712565009.5342155, + 1712565010.6368752, + {} + ], + [ + 4, + null, + 209652396, + null, + 0.02560352831940571, + 1.2457053661346436, + 1, + 1712565009.573759, + 1712565010.8200808, + {} + ], + [ + 5, + null, + 209652396, + null, + 0.8992773135252243, + 10.674755573272705, + 1, + 1712565009.611309, + 1712565020.286695, + {} + ], + [ + 6, + null, + 209652396, + null, + 0.19250851129681212, + 0.5179076194763184, + 1, + 1712565009.6513555, + 1712565010.1701534, + {} + ], + [ + 7, + null, + 209652396, + null, + 0.89983286908078, + 1.9669127464294434, + 1, + 1712565009.704494, + 1712565011.67224, + {} + ], + [ + 8, + null, + 209652396, + null, + 0.12855926957598263, + 0.35997724533081055, + 1, + 1712565009.744616, + 1712565010.1052234, + {} + ], + [ + 9, + null, + 209652396, + null, + 0.05064376354069944, + 2.6614692211151123, + 1, + 1712565009.8414927, + 1712565012.5038698, + {} + ], + [ + 1, + null, + 398764591, + null, + 0.6861915815536985, + 0.3299448490142822, + 1, + 1712565010.1649523, + 1712565010.4956682, + {} + ], + [ + 10, + null, + 209652396, + null, + 0.2904967502321263, + 0.49783873558044434, + 1, + 1712565010.2330816, + 1712565010.7317798, + {} + ], + [ + 8, + null, + 398764591, + null, + 0.09072887650882078, + 0.3993697166442871, + 1, + 1712565010.5594473, + 1712565010.959439, + {} + ], + [ + 11, + null, + 209652396, + null, + 0.030058805323429305, + 8.363003253936768, + 1, + 1712565010.6421216, + 1712565019.005703, + {} + ], + [ + 2, + null, + 398764591, + null, + 0.03006035283194064, + 1.5795397758483887, + 1, + 1712565010.7179625, + 1712565012.2989554, + {} + ], + [ + 12, + null, + 209652396, + null, + 0.4312689569792634, + 0.41001152992248535, + 1, + 1712565010.7946854, + 1712565011.2055905, + {} + ], + [ + 13, + null, + 209652396, + null, + 0.8987186629526462, + 0.787147045135498, + 1, + 1712565010.8761115, + 1712565011.6642888, + {} + ], + [ + 14, + null, + 209652396, + null, + 0.02392757660167144, + 0.7371306419372559, + 1, + 1712565011.0186121, + 1712565011.7567017, + {} + ], + [ + 15, + null, + 209652396, + null, + 0.27155834107087595, + 0.5496902465820312, + 1, + 1712565011.2593076, + 1712565011.8098226, + {} + ], + [ + 16, + null, + 209652396, + null, + 0.027825750541627947, + 2.965526819229126, + 1, + 1712565011.714806, + 1712565014.681264, + {} + ], + [ + 17, + null, + 209652396, + null, + 0.2031012070566388, + 0.9088141918182373, + 1, + 1712565011.782877, + 1712565012.692426, + {} + ], + [ + 14, + null, + 398764591, + null, + 0.03616527390900648, + 0.5853779315948486, + 1, + 1712565011.84823, + 1712565012.43451, + {} + ], + [ + 18, + null, + 209652396, + null, + 0.043975549365521616, + 1.1089684963226318, + 1, + 1712565011.9250941, + 1712565013.034664, + {} + ], + [ + 19, + null, + 209652396, + null, + 0.0473026926648098, + 0.8433613777160645, + 1, + 1712565012.3523204, + 1712565013.196589, + {} + ], + [ + 20, + null, + 209652396, + null, + 0.019481584648715633, + 1.379931926727295, + 1, + 1712565012.4961815, + 1712565013.877354, + {} + ], + [ + 14, + null, + 924231285, + null, + 0.034498607242339774, + 0.5644114017486572, + 1, + 1712565012.5570464, + 1712565013.1220315, + {} + ], + [ + 9, + null, + 398764591, + null, + 0.046202414113277634, + 3.3080027103424072, + 1, + 1712565012.7521577, + 1712565016.0607708, + {} + ], + [ + 17, + null, + 398764591, + null, + 0.2303899721448468, + 0.8374693393707275, + 1, + 1712565013.0987794, + 1712565013.9371634, + {} + ], + [ + 18, + null, + 398764591, + null, + 0.04397709687403284, + 1.4642667770385742, + 1, + 1712565013.1837516, + 1712565014.6489024, + {} + ], + [ + 21, + null, + 209652396, + null, + 0.8102460538532962, + 1.399742841720581, + 1, + 1712565013.24768, + 1712565014.6482792, + {} + ], + [ + 19, + null, + 398764591, + null, + 0.04674558960074293, + 0.9420022964477539, + 1, + 1712565013.938809, + 1712565014.881975, + {} + ], + [ + 20, + null, + 398764591, + null, + 0.023378211080160916, + 1.4037482738494873, + 1, + 1712565014.003451, + 1712565015.4077935, + {} + ], + [ + 17, + null, + 924231285, + null, + 0.2531925100588053, + 0.8451352119445801, + 1, + 1712565014.7065277, + 1712565015.5526712, + {} + ], + [ + 16, + null, + 398764591, + null, + 0.02560662333642827, + 3.018314838409424, + 1, + 1712565014.7788284, + 1712565017.7980478, + {} + ], + [ + 18, + null, + 924231285, + null, + 0.033389043639740046, + 1.1432137489318848, + 1, + 1712565014.83027, + 1712565015.9741292, + {} + ], + [ + 21, + null, + 398764591, + null, + 0.8770318786753327, + 1.9530634880065918, + 1, + 1712565014.9338968, + 1712565016.8878474, + {} + ], + [ + 19, + null, + 924231285, + null, + 0.07066542865985759, + 1.0578465461730957, + 1, + 1712565015.468166, + 1712565016.5266423, + {} + ], + [ + 20, + null, + 924231285, + null, + 0.02059114825131536, + 2.661302089691162, + 1, + 1712565015.615147, + 1712565018.2773883, + {} + ], + [ + 22, + null, + 209652396, + null, + 0.04287062828845567, + 1.5211269855499268, + 1, + 1712565016.0558727, + 1712565017.5779333, + {} + ], + [ + 23, + null, + 209652396, + null, + 0.04841999380996587, + 8.354167461395264, + 1, + 1712565016.1641889, + 1712565024.5193043, + {} + ], + [ + 9, + null, + 924231285, + null, + 0.0456313834726092, + 3.8456690311431885, + 1, + 1712565016.6118283, + 1712565020.459012, + {} + ], + [ + 24, + null, + 209652396, + null, + 0.8008805323429279, + 0.46524930000305176, + 1, + 1712565016.9718251, + 1712565017.438463, + {} + ], + [ + 21, + null, + 924231285, + null, + 0.8920519962859795, + 2.354919672012329, + 1, + 1712565017.5199628, + 1712565019.8761735, + {} + ], + [ + 24, + null, + 398764591, + null, + 0.726819870009285, + 0.6692235469818115, + 1, + 1712565017.6583474, + 1712565018.328519, + {} + ], + [ + 22, + null, + 398764591, + null, + 0.032836583101207184, + 1.1873645782470703, + 1, + 1712565017.88121, + 1712565019.0695643, + {} + ], + [ + 16, + null, + 924231285, + null, + 0.02894769421231813, + 3.8686747550964355, + 1, + 1712565018.3620129, + 1712565022.2322352, + {} + ], + [ + 24, + null, + 924231285, + null, + 0.7039430516867843, + 0.5134823322296143, + 1, + 1712565018.45915, + 1712565018.9735632, + {} + ], + [ + 25, + null, + 209652396, + null, + 0.0473026926648098, + 16.41287326812744, + 1, + 1712565019.0678082, + 1712565035.4815993, + {} + ], + [ + 11, + null, + 398764591, + null, + 0.032836583101207184, + 11.42868709564209, + 1, + 1712565019.1586893, + 1712565030.5886083, + {} + ], + [ + 22, + null, + 924231285, + null, + 0.042299597647787124, + 1.5826566219329834, + 1, + 1712565019.2559285, + 1712565020.8400044, + {} + ], + [ + 26, + null, + 209652396, + null, + 0.7974450634478489, + 0.5728342533111572, + 1, + 1712565019.9510424, + 1712565020.5249114, + {} + ], + [ + 27, + null, + 209652396, + null, + 0.09015010832559567, + 4.364055395126343, + 1, + 1712565020.3812418, + 1712565024.7467766, + {} + ], + [ + 5, + null, + 398764591, + null, + 0.9003884246363355, + 14.183544158935547, + 1, + 1712565020.5460055, + 1712565034.730529, + {} + ], + [ + 28, + null, + 209652396, + null, + 0.0984989167440421, + 0.9867184162139893, + 1, + 1712565020.6601338, + 1712565021.6482403, + {} + ], + [ + 26, + null, + 398764591, + null, + 0.6844769421231817, + 0.5313403606414795, + 1, + 1712565020.927448, + 1712565021.4603171, + {} + ], + [ + 29, + null, + 209652396, + null, + 0.8981615598885794, + 0.22849583625793457, + 1, + 1712565021.561606, + 1712565021.7911196, + {} + ], + [ + 26, + null, + 924231285, + null, + 0.823571649644073, + 0.731696605682373, + 1, + 1712565021.7452338, + 1712565022.4780235, + {} + ], + [ + 28, + null, + 398764591, + null, + 0.10796193129062215, + 1.0691678524017334, + 1, + 1712565021.888155, + 1712565022.9592311, + {} + ], + [ + 29, + null, + 398764591, + null, + 0.89983286908078, + 0.24839043617248535, + 1, + 1712565022.327761, + 1712565022.57766, + {} + ], + [ + 30, + null, + 209652396, + null, + 0.8992742185082018, + 10.086802959442139, + 1, + 1712565022.5642378, + 1712565032.651993, + {} + ], + [ + 31, + null, + 209652396, + null, + 0.1001562983596409, + 1.2912817001342773, + 1, + 1712565022.6596029, + 1712565023.9524426, + {} + ], + [ + 29, + null, + 924231285, + null, + 0.9059610027855153, + 0.22541332244873047, + 1, + 1712565023.0410864, + 1712565023.268075, + {} + ], + [ + 28, + null, + 924231285, + null, + 0.09126121943670695, + 1.0557401180267334, + 1, + 1712565023.348831, + 1712565024.4059231, + {} + ], + [ + 32, + null, + 209652396, + null, + 0.14357319715258432, + 0.7567057609558105, + 1, + 1712565024.0348003, + 1712565024.7930315, + {} + ], + [ + 31, + null, + 398764591, + null, + 0.12243268337975866, + 0.9943161010742188, + 1, + 1712565024.4760678, + 1712565025.4713368, + {} + ], + [ + 33, + null, + 209652396, + null, + 0.16188022284122572, + 25.683335304260254, + 1, + 1712565024.600618, + 1712565050.285301, + {} + ], + [ + 23, + null, + 398764591, + null, + 0.051753327143299166, + 7.385228157043457, + 1, + 1712565024.8224359, + 1712565032.2086697, + {} + ], + [ + 27, + null, + 398764591, + null, + 0.1191024450634478, + 3.959066867828369, + 1, + 1712565024.9297316, + 1712565028.8901627, + {} + ], + [ + 32, + null, + 398764591, + null, + 0.135792324357784, + 0.838543176651001, + 1, + 1712565025.5465531, + 1712565026.3867607, + {} + ], + [ + 31, + null, + 924231285, + null, + 0.10852986691426803, + 1.264732837677002, + 1, + 1712565026.4774003, + 1712565027.743719, + {} + ], + [ + 32, + null, + 924231285, + null, + 0.1413370473537604, + 0.9844629764556885, + 1, + 1712565027.8312953, + 1712565028.8173516, + {} + ], + [ + 34, + null, + 209652396, + null, + 0.14081089445991957, + 0.7884180545806885, + 1, + 1712565028.910618, + 1712565029.700526, + {} + ], + [ + 27, + null, + 924231285, + null, + 0.10739863819251005, + 4.667412996292114, + 1, + 1712565029.015657, + 1712565033.6845143, + {} + ], + [ + 35, + null, + 209652396, + null, + 0.04230114515629835, + 3.2498931884765625, + 1, + 1712565029.786574, + 1712565033.0380797, + {} + ], + [ + 34, + null, + 398764591, + null, + 0.16361188486536682, + 0.6655764579772949, + 1, + 1712565030.678432, + 1712565031.34571, + {} + ], + [ + 11, + null, + 924231285, + null, + 0.03283194057567318, + 12.322223424911499, + 1, + 1712565031.4208908, + 1712565043.7444239, + {} + ], + [ + 34, + null, + 924231285, + null, + 0.10627050448777475, + 0.6005818843841553, + 1, + 1712565032.2954042, + 1712565032.8974326, + {} + ], + [ + 23, + null, + 924231285, + null, + 0.06121324667285677, + 7.619780540466309, + 1, + 1712565032.744003, + 1712565040.3652062, + {} + ], + [ + 30, + null, + 398764591, + null, + 0.8948297740637574, + 11.035215616226196, + 1, + 1712565032.9881554, + 1712565044.0248187, + {} + ], + [ + 36, + null, + 209652396, + null, + 0.33456979263385933, + 0.9639077186584473, + 1, + 1712565033.1345205, + 1712565034.1002426, + {} + ], + [ + 35, + null, + 398764591, + null, + 0.04396007428040849, + 3.2995266914367676, + 1, + 1712565033.7999034, + 1712565037.1014593, + {} + ], + [ + 37, + null, + 209652396, + null, + 0.8992773135252244, + 0.7176535129547119, + 1, + 1712565034.1942468, + 1712565034.9135132, + {} + ], + [ + 36, + null, + 398764591, + null, + 0.5536861652739089, + 0.7682046890258789, + 1, + 1712565034.8139176, + 1712565035.583699, + {} + ], + [ + 5, + null, + 924231285, + null, + 0.9003899721448468, + 14.44285535812378, + 1, + 1712565035.018067, + 1712565049.4626539, + {} + ], + [ + 37, + null, + 398764591, + null, + 0.8987202104611576, + 0.7350404262542725, + 1, + 1712565035.604154, + 1712565036.3408573, + {} + ], + [ + 25, + null, + 398764591, + null, + 0.04230114515629835, + 16.373348712921143, + 1, + 1712565035.7092667, + 1712565052.084024, + {} + ], + [ + 36, + null, + 924231285, + null, + 0.4841782729805014, + 0.6209244728088379, + 1, + 1712565036.4488175, + 1712565037.071179, + {} + ], + [ + 37, + null, + 924231285, + null, + 0.89983286908078, + 0.8147656917572021, + 1, + 1712565037.180723, + 1712565037.9965858, + {} + ], + [ + 35, + null, + 924231285, + null, + 0.03784122562674097, + 2.6988790035247803, + 1, + 1712565037.301929, + 1712565040.002422, + {} + ], + [ + 38, + null, + 209652396, + null, + 0.8992757660167131, + 2.005662679672241, + 1, + 1712565038.0986674, + 1712565040.10565, + {} + ], + [ + 39, + null, + 209652396, + null, + 0.10461931290622106, + 1.514883041381836, + 1, + 1712565040.0788782, + 1712565041.5947666, + {} + ], + [ + 40, + null, + 209652396, + null, + 0.023930671618693777, + 2.3705286979675293, + 1, + 1712565040.184661, + 1712565042.556817, + {} + ], + [ + 38, + null, + 398764591, + null, + 0.8987186629526462, + 1.8411645889282227, + 1, + 1712565040.4387646, + 1712565042.2815857, + {} + ], + [ + 41, + null, + 209652396, + null, + 0.5219870009285051, + 2.545016050338745, + 1, + 1712565041.699431, + 1712565044.2464228, + {} + ], + [ + 39, + null, + 398764591, + null, + 0.06066233364283513, + 1.1961593627929688, + 1, + 1712565042.3568745, + 1712565043.5545325, + {} + ], + [ + 38, + null, + 924231285, + null, + 0.8987186629526462, + 2.093280792236328, + 1, + 1712565042.63884, + 1712565044.7334743, + {} + ], + [ + 40, + null, + 398764591, + null, + 0.02560352831940571, + 2.676013469696045, + 1, + 1712565043.6318994, + 1712565046.3088646, + {} + ], + [ + 39, + null, + 924231285, + null, + 0.07399566697616833, + 1.149909496307373, + 1, + 1712565043.8234403, + 1712565044.9747822, + {} + ], + [ + 42, + null, + 209652396, + null, + 0.041186939028164615, + 1.3158142566680908, + 1, + 1712565044.1248553, + 1712565045.4421346, + {} + ], + [ + 30, + null, + 924231285, + null, + 0.9009470752089137, + 10.608931303024292, + 1, + 1712565044.3304265, + 1712565054.9409497, + {} + ], + [ + 41, + null, + 398764591, + null, + 0.4590931600123801, + 2.614492177963257, + 1, + 1712565044.812999, + 1712565047.429369, + {} + ], + [ + 43, + null, + 209652396, + null, + 0.12633859486227172, + 1.004054069519043, + 1, + 1712565045.064711, + 1712565046.0705123, + {} + ], + [ + 44, + null, + 209652396, + null, + 0.15694367069018877, + 6.814570188522339, + 1, + 1712565045.5238674, + 1712565052.3401663, + {} + ], + [ + 42, + null, + 398764591, + null, + 0.03673320953265247, + 1.7806165218353271, + 1, + 1712565046.185416, + 1712565047.967166, + {} + ], + [ + 43, + null, + 398764591, + null, + 0.16193283813060977, + 1.155543565750122, + 1, + 1712565046.4128385, + 1712565047.570328, + {} + ], + [ + 40, + null, + 924231285, + null, + 0.024484679665738307, + 2.6976280212402344, + 1, + 1712565047.529584, + 1712565050.2290654, + {} + ], + [ + 41, + null, + 924231285, + null, + 0.3995605075827917, + 1.997565507888794, + 1, + 1712565047.6774108, + 1712565049.6766438, + {} + ], + [ + 43, + null, + 924231285, + null, + 0.13798978644382542, + 0.9540266990661621, + 1, + 1712565048.050342, + 1712565049.0060203, + {} + ], + [ + 42, + null, + 924231285, + null, + 0.03227793252862887, + 1.4512040615081787, + 1, + 1712565049.0885086, + 1712565050.5413556, + {} + ], + [ + 45, + null, + 209652396, + null, + 0.8753203342618384, + 0.847496747970581, + 1, + 1712565049.543191, + 1712565050.392285, + {} + ], + [ + 46, + null, + 209652396, + null, + 0.2489322191272052, + 0.6393148899078369, + 1, + 1712565049.7643716, + 1712565050.4047034, + {} + ], + [ + 47, + null, + 209652396, + null, + 0.08013927576601676, + 15.927346467971802, + 1, + 1712565050.3042223, + 1712565066.232525, + {} + ], + [ + 33, + null, + 398764591, + null, + 0.1925379139585267, + 23.66932439804077, + 1, + 1712565050.3959663, + 1712565074.0666122, + {} + ], + [ + 48, + null, + 209652396, + null, + 0.8124187558031569, + 0.910470724105835, + 1, + 1712565050.4891617, + 1712565051.4010818, + {} + ], + [ + 45, + null, + 398764591, + null, + 0.8914763231197772, + 0.7565186023712158, + 1, + 1712565050.623894, + 1712565051.3820372, + {} + ], + [ + 46, + null, + 398764591, + null, + 0.35610492107706604, + 0.8270492553710938, + 1, + 1712565050.7091923, + 1712565051.5371573, + {} + ], + [ + 49, + null, + 209652396, + null, + 0.2715707211389663, + 6.957829475402832, + 1, + 1712565051.466912, + 1712565058.4263031, + {} + ], + [ + 48, + null, + 398764591, + null, + 0.764634787991334, + 0.8151421546936035, + 1, + 1712565051.5597835, + 1712565052.3764203, + {} + ], + [ + 45, + null, + 924231285, + null, + 0.8747632311977716, + 0.941584587097168, + 1, + 1712565051.6266928, + 1712565052.5692742, + {} + ], + [ + 46, + null, + 924231285, + null, + 0.10686939028164644, + 0.5962722301483154, + 1, + 1712565052.165133, + 1712565052.762847, + {} + ], + [ + 25, + null, + 924231285, + null, + 0.04730424017332102, + 16.94296383857727, + 1, + 1712565052.4514196, + 1712565069.3960361, + {} + ], + [ + 44, + null, + 398764591, + null, + 0.18422624574435154, + 6.634417295455933, + 1, + 1712565052.544118, + 1712565059.1801126, + {} + ], + [ + 48, + null, + 924231285, + null, + 0.8602785515320335, + 0.7708423137664795, + 1, + 1712565052.6536405, + 1712565053.4260578, + {} + ], + [ + 50, + null, + 209652396, + null, + 0.04842463633549987, + 0.9821035861968994, + 1, + 1712565052.8549998, + 1712565053.8382864, + {} + ], + [ + 51, + null, + 209652396, + null, + 0.03673475704116369, + 10.69734239578247, + 1, + 1712565053.5026176, + 1712565064.2009585, + {} + ], + [ + 52, + null, + 209652396, + null, + 0.9031723924481585, + 0.40129661560058594, + 1, + 1712565053.932928, + 1712565054.335735, + {} + ], + [ + 50, + null, + 398764591, + null, + 0.04452955741256592, + 0.7838237285614014, + 1, + 1712565054.4255831, + 1712565055.2110846, + {} + ], + [ + 52, + null, + 398764591, + null, + 0.9031723924481585, + 0.40575075149536133, + 1, + 1712565055.0314689, + 1712565055.4387197, + {} + ], + [ + 53, + null, + 209652396, + null, + 0.8987202104611576, + 0.95816969871521, + 1, + 1712565055.2899451, + 1712565056.2494361, + {} + ], + [ + 50, + null, + 924231285, + null, + 0.04896626431445372, + 0.717515230178833, + 1, + 1712565055.52647, + 1712565056.2451823, + {} + ], + [ + 52, + null, + 924231285, + null, + 0.9037264004952027, + 0.4451889991760254, + 1, + 1712565056.3189044, + 1712565056.7654727, + {} + ], + [ + 53, + null, + 398764591, + null, + 0.8992757660167131, + 1.097594976425171, + 1, + 1712565056.4288678, + 1712565057.5279367, + {} + ], + [ + 54, + null, + 209652396, + null, + 0.08237233054781812, + 1.8947386741638184, + 1, + 1712565056.8644137, + 1712565058.76014, + {} + ], + [ + 55, + null, + 209652396, + null, + 0.0801470133085731, + 1.0197317600250244, + 1, + 1712565057.613952, + 1712565058.6350822, + {} + ], + [ + 53, + null, + 924231285, + null, + 0.888133704735376, + 0.9786050319671631, + 1, + 1712565058.522568, + 1712565059.502105, + {} + ], + [ + 49, + null, + 398764591, + null, + 0.5036165273909007, + 7.000282049179077, + 1, + 1712565058.730699, + 1712565065.7326648, + {} + ], + [ + 55, + null, + 398764591, + null, + 0.08347415660786128, + 0.8997397422790527, + 1, + 1712565058.8603768, + 1712565059.7616763, + {} + ], + [ + 54, + null, + 398764591, + null, + 0.08682141751779626, + 2.5480194091796875, + 1, + 1712565059.2588532, + 1712565061.808606, + {} + ], + [ + 44, + null, + 924231285, + null, + 0.1307644692045805, + 7.21750807762146, + 1, + 1712565059.5947275, + 1712565066.8132136, + {} + ], + [ + 56, + null, + 209652396, + null, + 0.024493964716805983, + 2.156345844268799, + 1, + 1712565059.8427088, + 1712565062.0000536, + {} + ], + [ + 55, + null, + 924231285, + null, + 0.06342927886103378, + 0.9959497451782227, + 1, + 1712565061.8952117, + 1712565062.892992, + {} + ], + [ + 54, + null, + 924231285, + null, + 0.08180903744970591, + 2.1299283504486084, + 1, + 1712565062.0780683, + 1712565064.2096357, + {} + ], + [ + 56, + null, + 398764591, + null, + 0.027270194986072527, + 2.475947856903076, + 1, + 1712565062.9701085, + 1712565065.4473305, + {} + ], + [ + 57, + null, + 209652396, + null, + 0.8281089445991954, + 0.6765487194061279, + 1, + 1712565064.2821856, + 1712565064.9597, + {} + ], + [ + 51, + null, + 398764591, + null, + 0.04229495512225334, + 10.195435047149658, + 1, + 1712565064.3825853, + 1712565074.5789762, + {} + ], + [ + 58, + null, + 209652396, + null, + 0.8280609718353451, + 7.688541412353516, + 1, + 1712565065.0443525, + 1712565072.7342741, + {} + ], + [ + 57, + null, + 398764591, + null, + 0.8636691426802847, + 1.2278459072113037, + 1, + 1712565065.5273557, + 1712565066.756225, + {} + ], + [ + 56, + null, + 924231285, + null, + 0.02893995666976168, + 2.2545557022094727, + 1, + 1712565065.8200572, + 1712565068.075699, + {} + ], + [ + 49, + null, + 924231285, + null, + 0.48469978334880837, + 6.26426362991333, + 1, + 1712565066.338829, + 1712565072.6052105, + {} + ], + [ + 47, + null, + 398764591, + null, + 0.08179975239863813, + 9.90377926826477, + 1, + 1712565066.839424, + 1712565076.744739, + {} + ], + [ + 57, + null, + 924231285, + null, + 0.8580532342927887, + 0.9290714263916016, + 1, + 1712565066.9271002, + 1712565067.857914, + {} + ], + [ + 59, + null, + 209652396, + null, + 0.07180594243268335, + 0.8639552593231201, + 1, + 1712565067.9361448, + 1712565068.801029, + {} + ], + [ + 60, + null, + 209652396, + null, + 0.025601980810894487, + 1.736633539199829, + 1, + 1712565068.1614459, + 1712565069.8996112, + {} + ], + [ + 61, + null, + 209652396, + null, + 0.7572144846796658, + 4.349518537521362, + 1, + 1712565068.9006317, + 1712565073.2513459, + {} + ], + [ + 59, + null, + 398764591, + null, + 0.09125502940266172, + 1.0945045948028564, + 1, + 1712565069.482275, + 1712565070.5787363, + {} + ] + ], + "configs": { + "1": { + "activation": "logistic", + "n_layer": 4, + "n_neurons": 11, + "solver": "lbfgs" + }, + "2": { + "activation": "tanh", + "n_layer": 1, + "n_neurons": 146, + "solver": "adam", + "batch_size": 77, + "learning_rate_init": 0.000534923804864797 + }, + "3": { + "activation": "relu", + "n_layer": 5, + "n_neurons": 43, + "solver": "sgd", + "batch_size": 157, + "learning_rate": "invscaling", + "learning_rate_init": 0.08903539757637764 + }, + "4": { + "activation": "logistic", + "n_layer": 2, + "n_neurons": 90, + "solver": "adam", + "batch_size": 175, + "learning_rate_init": 0.004156370184967407 + }, + "5": { + "activation": "logistic", + "n_layer": 4, + "n_neurons": 200, + "solver": "sgd", + "batch_size": 204, + "learning_rate": "constant", + "learning_rate_init": 0.016071068084706618 + }, + "6": { + "activation": "relu", + "n_layer": 2, + "n_neurons": 12, + "solver": "sgd", + "batch_size": 119, + "learning_rate": "adaptive", + "learning_rate_init": 0.0022428359458905173 + }, + "7": { + "activation": "relu", + "n_layer": 4, + "n_neurons": 51, + "solver": "adam", + "batch_size": 31, + "learning_rate_init": 0.11617003217050809 + }, + "8": { + "activation": "tanh", + "n_layer": 2, + "n_neurons": 20, + "solver": "lbfgs" + }, + "9": { + "activation": "tanh", + "n_layer": 5, + "n_neurons": 61, + "solver": "lbfgs" + }, + "10": { + "activation": "tanh", + "n_layer": 3, + "n_neurons": 26, + "solver": "sgd", + "batch_size": 233, + "learning_rate": "adaptive", + "learning_rate_init": 0.001104760089338138 + }, + "11": { + "activation": "relu", + "n_layer": 3, + "n_neurons": 214, + "solver": "lbfgs" + }, + "12": { + "activation": "logistic", + "n_layer": 1, + "n_neurons": 18, + "solver": "adam", + "batch_size": 135, + "learning_rate_init": 0.00015265008040823863 + }, + "13": { + "activation": "logistic", + "n_layer": 4, + "n_neurons": 29, + "solver": "adam", + "batch_size": 106, + "learning_rate_init": 0.33335905507564456 + }, + "14": { + "activation": "relu", + "n_layer": 1, + "n_neurons": 84, + "solver": "lbfgs" + }, + "15": { + "activation": "tanh", + "n_layer": 5, + "n_neurons": 8, + "solver": "adam", + "batch_size": 279, + "learning_rate_init": 0.04285190354891313 + }, + "16": { + "activation": "tanh", + "n_layer": 2, + "n_neurons": 122, + "solver": "sgd", + "batch_size": 61, + "learning_rate": "constant", + "learning_rate_init": 0.00630547594663167 + }, + "17": { + "activation": "tanh", + "n_layer": 5, + "n_neurons": 23, + "solver": "adam", + "batch_size": 172, + "learning_rate_init": 0.00040121151561322107 + }, + "18": { + "activation": "tanh", + "n_layer": 2, + "n_neurons": 66, + "solver": "lbfgs" + }, + "19": { + "activation": "relu", + "n_layer": 4, + "n_neurons": 16, + "solver": "sgd", + "batch_size": 66, + "learning_rate": "constant", + "learning_rate_init": 0.005542589894513166 + }, + "20": { + "activation": "logistic", + "n_layer": 1, + "n_neurons": 232, + "solver": "sgd", + "batch_size": 253, + "learning_rate": "adaptive", + "learning_rate_init": 0.06675517414146671 + }, + "21": { + "activation": "logistic", + "n_layer": 3, + "n_neurons": 74, + "solver": "sgd", + "batch_size": 300, + "learning_rate": "adaptive", + "learning_rate_init": 0.0029904795631048244 + }, + "22": { + "activation": "relu", + "n_layer": 1, + "n_neurons": 32, + "solver": "adam", + "batch_size": 45, + "learning_rate_init": 0.012048758384365402 + }, + "23": { + "activation": "relu", + "n_layer": 5, + "n_neurons": 106, + "solver": "lbfgs" + }, + "24": { + "activation": "tanh", + "n_layer": 3, + "n_neurons": 9, + "solver": "adam", + "batch_size": 210, + "learning_rate_init": 0.1549368466139337 + }, + "25": { + "activation": "tanh", + "n_layer": 4, + "n_neurons": 158, + "solver": "sgd", + "batch_size": 143, + "learning_rate": "adaptive", + "learning_rate_init": 0.00147326235260944 + }, + "26": { + "activation": "relu", + "n_layer": 2, + "n_neurons": 10, + "solver": "sgd", + "batch_size": 193, + "learning_rate": "invscaling", + "learning_rate_init": 0.018836814953733322 + }, + "27": { + "activation": "relu", + "n_layer": 4, + "n_neurons": 97, + "solver": "adam", + "batch_size": 249, + "learning_rate_init": 0.00011444921049693927 + }, + "28": { + "activation": "logistic", + "n_layer": 2, + "n_neurons": 38, + "solver": "lbfgs" + }, + "29": { + "activation": "logistic", + "n_layer": 5, + "n_neurons": 13, + "solver": "lbfgs" + }, + "30": { + "activation": "relu", + "n_layer": 3, + "n_neurons": 175, + "solver": "adam", + "batch_size": 271, + "learning_rate_init": 0.4444919521413565 + }, + "31": { + "activation": "tanh", + "n_layer": 3, + "n_neurons": 21, + "solver": "lbfgs" + }, + "32": { + "activation": "logistic", + "n_layer": 1, + "n_neurons": 45, + "solver": "sgd", + "batch_size": 106, + "learning_rate": "invscaling", + "learning_rate_init": 0.03214161586827769 + }, + "33": { + "activation": "logistic", + "n_layer": 5, + "n_neurons": 224, + "solver": "adam", + "batch_size": 290, + "learning_rate_init": 0.004563292654888671 + }, + "34": { + "activation": "tanh", + "n_layer": 2, + "n_neurons": 17, + "solver": "lbfgs" + }, + "35": { + "activation": "relu", + "n_layer": 4, + "n_neurons": 57, + "solver": "adam", + "batch_size": 116, + "learning_rate_init": 0.00035489292891332955 + }, + "36": { + "activation": "logistic", + "n_layer": 1, + "n_neurons": 27, + "solver": "sgd", + "batch_size": 203, + "learning_rate": "adaptive", + "learning_rate_init": 0.7609797301059412 + }, + "37": { + "activation": "logistic", + "n_layer": 4, + "n_neurons": 8, + "solver": "sgd", + "batch_size": 182, + "learning_rate": "adaptive", + "learning_rate_init": 0.00019717209149247594 + }, + "38": { + "activation": "relu", + "n_layer": 2, + "n_neurons": 130, + "solver": "sgd", + "batch_size": 163, + "learning_rate": "constant", + "learning_rate_init": 0.14078837052759072 + }, + "39": { + "activation": "relu", + "n_layer": 4, + "n_neurons": 31, + "solver": "lbfgs" + }, + "40": { + "activation": "tanh", + "n_layer": 2, + "n_neurons": 80, + "solver": "adam", + "batch_size": 260, + "learning_rate_init": 0.01019194380084181 + }, + "41": { + "activation": "tanh", + "n_layer": 3, + "n_neurons": 40, + "solver": "sgd", + "batch_size": 58, + "learning_rate": "adaptive", + "learning_rate_init": 0.00012594180136887576 + }, + "42": { + "activation": "tanh", + "n_layer": 1, + "n_neurons": 94, + "solver": "lbfgs" + }, + "43": { + "activation": "relu", + "n_layer": 5, + "n_neurons": 12, + "solver": "adam", + "batch_size": 232, + "learning_rate_init": 0.0017417895884476993 + }, + "44": { + "activation": "logistic", + "n_layer": 3, + "n_neurons": 137, + "solver": "lbfgs" + }, + "45": { + "activation": "logistic", + "n_layer": 5, + "n_neurons": 54, + "solver": "lbfgs" + }, + "46": { + "activation": "relu", + "n_layer": 2, + "n_neurons": 19, + "solver": "adam", + "batch_size": 185, + "learning_rate_init": 0.039036592000729606 + }, + "47": { + "activation": "tanh", + "n_layer": 4, + "n_neurons": 191, + "solver": "sgd", + "batch_size": 240, + "learning_rate": "constant", + "learning_rate_init": 0.0007368727861046234 + }, + "48": { + "activation": "tanh", + "n_layer": 1, + "n_neurons": 13, + "solver": "sgd", + "batch_size": 88, + "learning_rate": "adaptive", + "learning_rate_init": 0.5025399902870405 + }, + "49": { + "activation": "logistic", + "n_layer": 4, + "n_neurons": 102, + "solver": "lbfgs" + }, + "50": { + "activation": "tanh", + "n_layer": 1, + "n_neurons": 35, + "solver": "adam", + "batch_size": 129, + "learning_rate_init": 0.0034213963211558564 + }, + "51": { + "activation": "relu", + "n_layer": 5, + "n_neurons": 148, + "solver": "lbfgs" + }, + "52": { + "activation": "logistic", + "n_layer": 2, + "n_neurons": 10, + "solver": "sgd", + "batch_size": 294, + "learning_rate": "constant", + "learning_rate_init": 0.00047328481860027055 + }, + "53": { + "activation": "logistic", + "n_layer": 5, + "n_neurons": 20, + "solver": "adam", + "batch_size": 256, + "learning_rate_init": 0.10560248678966794 + }, + "54": { + "activation": "relu", + "n_layer": 3, + "n_neurons": 48, + "solver": "sgd", + "batch_size": 68, + "learning_rate": "adaptive", + "learning_rate_init": 0.00026295202043791785 + }, + "55": { + "activation": "tanh", + "n_layer": 3, + "n_neurons": 14, + "solver": "adam", + "batch_size": 150, + "learning_rate_init": 0.013590087590816565 + }, + "56": { + "activation": "tanh", + "n_layer": 1, + "n_neurons": 167, + "solver": "lbfgs" + }, + "57": { + "activation": "tanh", + "n_layer": 5, + "n_neurons": 15, + "solver": "sgd", + "batch_size": 99, + "learning_rate": "constant", + "learning_rate_init": 0.2144498574918249 + }, + "58": { + "activation": "relu", + "n_layer": 2, + "n_neurons": 243, + "solver": "sgd", + "batch_size": 217, + "learning_rate": "invscaling", + "learning_rate_init": 0.00016793749976650653 + }, + "59": { + "activation": "relu", + "n_layer": 4, + "n_neurons": 24, + "solver": "lbfgs" + }, + "60": { + "activation": "logistic", + "n_layer": 1, + "n_neurons": 62, + "solver": "adam", + "batch_size": 51, + "learning_rate_init": 0.0013063520380558128 + }, + "61": { + "activation": "logistic", + "n_layer": 3, + "n_neurons": 114, + "solver": "adam", + "batch_size": 90, + "learning_rate_init": 0.0520608529688821 + } + }, + "config_origins": { + "1": "Initial Design: Sobol", + "2": "Initial Design: Sobol", + "3": "Initial Design: Sobol", + "4": "Initial Design: Sobol", + "5": "Initial Design: Sobol", + "6": "Initial Design: Sobol", + "7": "Initial Design: Sobol", + "8": "Initial Design: Sobol", + "9": "Initial Design: Sobol", + "10": "Initial Design: Sobol", + "11": "Initial Design: Sobol", + "12": "Initial Design: Sobol", + "13": "Initial Design: Sobol", + "14": "Initial Design: Sobol", + "15": "Initial Design: Sobol", + "16": "Initial Design: Sobol", + "17": "Initial Design: Sobol", + "18": "Initial Design: Sobol", + "19": "Initial Design: Sobol", + "20": "Initial Design: Sobol", + "21": "Initial Design: Sobol", + "22": "Initial Design: Sobol", + "23": "Initial Design: Sobol", + "24": "Initial Design: Sobol", + "25": "Initial Design: Sobol", + "26": "Initial Design: Sobol", + "27": "Initial Design: Sobol", + "28": "Initial Design: Sobol", + "29": "Initial Design: Sobol", + "30": "Initial Design: Sobol", + "31": "Initial Design: Sobol", + "32": "Initial Design: Sobol", + "33": "Initial Design: Sobol", + "34": "Initial Design: Sobol", + "35": "Initial Design: Sobol", + "36": "Initial Design: Sobol", + "37": "Initial Design: Sobol", + "38": "Initial Design: Sobol", + "39": "Initial Design: Sobol", + "40": "Initial Design: Sobol", + "41": "Initial Design: Sobol", + "42": "Initial Design: Sobol", + "43": "Initial Design: Sobol", + "44": "Initial Design: Sobol", + "45": "Initial Design: Sobol", + "46": "Initial Design: Sobol", + "47": "Initial Design: Sobol", + "48": "Initial Design: Sobol", + "49": "Initial Design: Sobol", + "50": "Initial Design: Sobol", + "51": "Initial Design: Sobol", + "52": "Initial Design: Sobol", + "53": "Initial Design: Sobol", + "54": "Initial Design: Sobol", + "55": "Initial Design: Sobol", + "56": "Initial Design: Sobol", + "57": "Initial Design: Sobol", + "58": "Initial Design: Sobol", + "59": "Initial Design: Sobol", + "60": "Initial Design: Sobol", + "61": "Initial Design: Sobol" + } +} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/scenario 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/scenario 2.json new file mode 100644 index 00000000..d6ceef93 --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_1/scenario 2.json @@ -0,0 +1,155 @@ +{ + "name": "bcdb4a7cedc37b0caa608ab570d77c03", + "deterministic": false, + "objectives": "cost", + "crash_cost": Infinity, + "termination_cost_threshold": Infinity, + "walltime_limit": 60, + "cputime_limit": Infinity, + "trial_walltime_limit": null, + "trial_memory_limit": null, + "n_trials": 500, + "use_default_config": false, + "instances": null, + "instance_features": null, + "min_budget": null, + "max_budget": null, + "seed": 0, + "n_workers": 8, + "_meta": { + "facade": { + "name": "HyperparameterOptimizationFacade" + }, + "runner": { + "name": "DaskParallelRunner" + }, + "model": { + "name": "RandomForest", + "types": [ + 3, + 0, + 0, + 3, + 0, + 4, + 0 + ], + "bounds": [ + [ + 3, + NaN + ], + [ + 0, + 1.0 + ], + [ + 0, + 1.0 + ], + [ + 3, + NaN + ], + [ + -1.0, + 1.0 + ], + [ + 4, + NaN + ], + [ + -1.0, + 1.0 + ] + ], + "pca_components": 7, + "n_trees": 10, + "n_points_per_tree": -1, + "ratio_features": 1.0, + "min_samples_split": 2, + "min_samples_leaf": 1, + "max_depth": 1048576, + "eps_purity": 1e-08, + "max_nodes": 1048576, + "bootstrapping": true + }, + "acquisition_maximizer": { + "name": "LocalAndSortedRandomSearch", + "acquisition_function": { + "name": "EI", + "xi": 0.0, + "log": true + }, + "challengers": 10000, + "seed": 0, + "random_search": { + "name": "RandomSearch", + "acquisition_function": { + "name": "EI", + "xi": 0.0, + "log": true + }, + "challengers": 5000, + "seed": 0 + }, + "local_search": { + "name": "LocalSearch", + "acquisition_function": { + "name": "EI", + "xi": 0.0, + "log": true + }, + "challengers": 5000, + "seed": 0, + "max_steps": null, + "n_steps_plateau_walk": 10, + "vectorization_min_obtain": 2, + "vectorization_max_obtain": 64 + } + }, + "acquisition_function": { + "name": "EI", + "xi": 0.0, + "log": true + }, + "intensifier": { + "name": "Intensifier", + "max_incumbents": 10, + "seed": 0 + }, + "initial_design": { + "name": "SobolInitialDesign", + "n_configs": 70, + "n_configs_per_hyperparameter": 10, + "additional_configs": [], + "seed": 0 + }, + "random_design": { + "name": "ProbabilityRandomDesign", + "seed": 0, + "probability": 0.2 + }, + "runhistory_encoder": { + "name": "RunHistoryLogScaledEncoder", + "considered_states": [ + 1, + 2, + 4 + ], + "lower_budget_states": [], + "scale_percentage": 5, + "seed": 0 + }, + "multi_objective_algorithm": null, + "config_selector": { + "name": "ConfigSelector", + "retrain_after": 8, + "retries": 16, + "min_trials": 1 + }, + "version": "2.0.2" + }, + "output_directory": "smac3_output/bcdb4a7cedc37b0caa608ab570d77c03/0" +} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/configspace 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/configspace 2.json new file mode 100644 index 00000000..be8aa231 --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_2/configspace 2.json @@ -0,0 +1 @@ +{"name": null, "hyperparameters": [{"type": "categorical", "name": "activation", "choices": ["logistic", "tanh", "relu"], "weights": null, "default_value": "tanh", "meta": null}, {"type": "uniform_int", "name": "n_layer", "lower": 1, "upper": 5, "default_value": 1, "log": false, "meta": null}, {"type": "uniform_int", "name": "n_neurons", "lower": 8, "upper": 256, "default_value": 10, "log": true, "meta": null}, {"type": "categorical", "name": "solver", "choices": ["lbfgs", "sgd", "adam"], "weights": null, "default_value": "adam", "meta": null}, {"type": "uniform_int", "name": "batch_size", "lower": 30, "upper": 300, "default_value": 200, "log": false, "meta": null}, {"type": "categorical", "name": "learning_rate", "choices": ["constant", "invscaling", "adaptive"], "weights": null, "default_value": "constant", "meta": null}, {"type": "uniform_float", "name": "learning_rate_init", "lower": 0.0001, "upper": 1.0, "default_value": 0.001, "log": true, "meta": null}], "conditions": [{"type": "IN", "child": "batch_size", "parent": "solver", "values": ["sgd", "adam"]}, {"type": "EQ", "child": "learning_rate", "parent": "solver", "value": "sgd"}, {"type": "IN", "child": "learning_rate_init", "parent": "solver", "values": ["sgd", "adam"]}], "forbiddens": [], "python_module_version": "1.2.0", "format_version": 0.4} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/intensifier 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/intensifier 2.json new file mode 100644 index 00000000..57b6338e --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_2/intensifier 2.json @@ -0,0 +1,74 @@ +{ + "incumbent_ids": [ + 4 + ], + "rejected_config_ids": [ + 8, + 1, + 7, + 3, + 13, + 10, + 14, + 12, + 2, + 17, + 6, + 15, + 16, + 9 + ], + "incumbents_changed": 3, + "trajectory": [ + { + "config_ids": [ + 1 + ], + "costs": [ + 0.6699473847106159 + ], + "trial": 1, + "walltime": 0.7647695541381836 + }, + { + "config_ids": [ + 10 + ], + "costs": [ + 0.05601722892809243 + ], + "trial": 14, + "walltime": 5.185967445373535 + }, + { + "config_ids": [ + 4 + ], + "costs": [ + 0.02671515526668729 + ], + "trial": 28, + "walltime": 9.271684408187866 + } + ], + "state": { + "queue": [ + [ + 5, + 2 + ], + [ + 19, + 2 + ], + [ + 11, + 4 + ], + [ + 18, + 4 + ] + ] + } +} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/optimization 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/optimization 2.json new file mode 100644 index 00000000..445f0dfa --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_2/optimization 2.json @@ -0,0 +1,6 @@ +{ + "used_walltime": 15.669282674789429, + "used_target_function_walltime": 102.76791548728943, + "last_update": 1706277639.8103335, + "finished": true +} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/runhistory 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/runhistory 2.json new file mode 100644 index 00000000..5be2d9d8 --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_2/runhistory 2.json @@ -0,0 +1,766 @@ +{ + "stats": { + "submitted": 50, + "finished": 50, + "running": 0 + }, + "data": [ + [ + 1, + null, + 209652396, + null, + 0.6699473847106159, + 0.6026918888092041, + 1, + 1706277624.222565, + 1706277624.8264945, + {} + ], + [ + 2, + null, + 209652396, + null, + 0.025597338285360594, + 3.1097147464752197, + 1, + 1706277624.2967758, + 1706277627.407669, + {} + ], + [ + 3, + null, + 209652396, + null, + 0.6583333333333333, + 2.336819648742676, + 1, + 1706277624.3608909, + 1706277626.6990438, + {} + ], + [ + 4, + null, + 209652396, + null, + 0.02560352831940571, + 2.9831438064575195, + 1, + 1706277624.415114, + 1706277627.3992622, + {} + ], + [ + 5, + null, + 209652396, + null, + 0.8992773135252243, + 14.455097913742065, + 1, + 1706277624.4994059, + 1706277638.9561086, + {} + ], + [ + 6, + null, + 209652396, + null, + 0.3678195605075827, + 4.080667018890381, + 1, + 1706277624.5847695, + 1706277628.6664875, + {} + ], + [ + 7, + null, + 209652396, + null, + 0.8992757660167131, + 1.912712574005127, + 1, + 1706277624.6662445, + 1706277626.5800583, + {} + ], + [ + 8, + null, + 209652396, + null, + 0.8241210151655834, + 0.5521142482757568, + 1, + 1706277624.7443416, + 1706277625.2985408, + {} + ], + [ + 9, + null, + 209652396, + null, + 0.0317254719900959, + 5.659100532531738, + 1, + 1706277624.910818, + 1706277630.5716987, + {} + ], + [ + 1, + null, + 398764591, + null, + 0.6861915815536985, + 0.8483016490936279, + 1, + 1706277625.409777, + 1706277626.2611463, + {} + ], + [ + 10, + null, + 209652396, + null, + 0.056199319096254974, + 0.7128767967224121, + 1, + 1706277626.3443322, + 1706277627.058281, + {} + ], + [ + 1, + null, + 924231285, + null, + 0.7017641597028783, + 0.723477840423584, + 1, + 1706277626.6957843, + 1706277627.4203732, + {} + ], + [ + 7, + null, + 398764591, + null, + 0.8987202104611576, + 1.744448184967041, + 1, + 1706277626.7947023, + 1706277628.5401833, + {} + ], + [ + 3, + null, + 398764591, + null, + 0.8458000619003405, + 1.7513632774353027, + 1, + 1706277627.1429644, + 1706277628.896084, + {} + ], + [ + 10, + null, + 398764591, + null, + 0.056761064685855844, + 0.6708414554595947, + 1, + 1706277627.5016356, + 1706277628.1742969, + {} + ], + [ + 2, + null, + 398764591, + null, + 0.03006035283194064, + 3.3852593898773193, + 1, + 1706277627.64613, + 1706277631.0330572, + {} + ], + [ + 4, + null, + 398764591, + null, + 0.027273290003095085, + 2.520305633544922, + 1, + 1706277627.740861, + 1706277630.2628653, + {} + ], + [ + 11, + null, + 209652396, + null, + 0.03840916125038696, + 7.205214738845825, + 1, + 1706277628.2559705, + 1706277635.4626877, + {} + ], + [ + 10, + null, + 924231285, + null, + 0.05509130300216647, + 0.6167201995849609, + 1, + 1706277628.640889, + 1706277629.2592533, + {} + ], + [ + 7, + null, + 924231285, + null, + 0.9009470752089137, + 1.3385493755340576, + 1, + 1706277628.76265, + 1706277630.102818, + {} + ], + [ + 6, + null, + 398764591, + null, + 0.398457134014237, + 3.8393754959106445, + 1, + 1706277628.9893534, + 1706277632.830835, + {} + ], + [ + 3, + null, + 924231285, + null, + Infinity, + 0.0, + 2, + 1706277629.3351586, + 1706277630.2660835, + { + "traceback": "Traceback (most recent call last):\n File \"/home/skrebs/Projects/Software/SMAC3/smac/runner/target_function_runner.py\", line 184, in run\n rval = self(config_copy, target_function, kwargs)\n File \"/home/skrebs/Projects/Software/SMAC3/smac/runner/target_function_runner.py\", line 257, in __call__\n return algorithm(config, **algorithm_kwargs)\n File \"/home/skrebs/Projects/Software/SMAC3/examples/2_multi_fidelity/1_mlp_epochs.py\", line 107, in train\n score = cross_val_score(classifier, dataset.data, dataset.target, cv=cv, error_score=\"raise\")\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/model_selection/_validation.py\", line 562, in cross_val_score\n cv_results = cross_validate(\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/utils/_param_validation.py\", line 211, in wrapper\n return func(*args, **kwargs)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/model_selection/_validation.py\", line 309, in cross_validate\n results = parallel(\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/utils/parallel.py\", line 65, in __call__\n return super().__call__(iterable_with_config)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 1088, in __call__\n while self.dispatch_one_batch(iterator):\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 901, in dispatch_one_batch\n self._dispatch(tasks)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 819, in _dispatch\n job = self._backend.apply_async(batch, callback=cb)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/_parallel_backends.py\", line 208, in apply_async\n result = ImmediateResult(func)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/_parallel_backends.py\", line 597, in __init__\n self.results = batch()\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 288, in __call__\n return [func(*args, **kwargs)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 288, in \n return [func(*args, **kwargs)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/utils/parallel.py\", line 127, in __call__\n return self.function(*args, **kwargs)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/model_selection/_validation.py\", line 732, in _fit_and_score\n estimator.fit(X_train, y_train, **fit_params)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/base.py\", line 1151, in wrapper\n return fit_method(estimator, *args, **kwargs)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py\", line 753, in fit\n return self._fit(X, y, incremental=False)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py\", line 496, in _fit\n raise ValueError(\nValueError: Solver produced non-finite parameter weights. The input data may contain large values and need to be preprocessed.\n", + "error": "ValueError('Solver produced non-finite parameter weights. The input data may contain large values and need to be preprocessed.')" + } + ], + [ + 12, + null, + 209652396, + null, + 0.10016403590219747, + 1.307952642440796, + 1, + 1706277630.1908402, + 1706277631.4997926, + {} + ], + [ + 13, + null, + 209652396, + null, + 0.89983286908078, + 0.2807650566101074, + 1, + 1706277630.3577998, + 1706277630.6401331, + {} + ], + [ + 4, + null, + 924231285, + null, + 0.02726864747756108, + 2.861912250518799, + 1, + 1706277630.4543989, + 1706277633.3173153, + {} + ], + [ + 14, + null, + 209652396, + null, + 0.3895543175487466, + 0.808574914932251, + 1, + 1706277630.6644979, + 1706277631.4741547, + {} + ], + [ + 9, + null, + 398764591, + null, + 0.032271742494583755, + 5.011699914932251, + 1, + 1706277630.76806, + 1706277635.7808084, + {} + ], + [ + 13, + null, + 398764591, + null, + 0.8886908077994429, + 0.32169437408447266, + 1, + 1706277631.1505885, + 1706277631.4740863, + {} + ], + [ + 2, + null, + 924231285, + null, + 0.029497059733828435, + 2.967978000640869, + 1, + 1706277631.5600839, + 1706277634.529425, + {} + ], + [ + 12, + null, + 398764591, + null, + 0.08014391829155054, + 1.0976686477661133, + 1, + 1706277631.6530569, + 1706277632.7523105, + {} + ], + [ + 14, + null, + 398764591, + null, + 0.495255338904364, + 0.6123650074005127, + 1, + 1706277631.7122657, + 1706277632.3262582, + {} + ], + [ + 13, + null, + 924231285, + null, + 0.9048467966573817, + 0.22585248947143555, + 1, + 1706277632.4026914, + 1706277632.6302176, + {} + ], + [ + 14, + null, + 924231285, + null, + 0.31948467966573824, + 0.7365078926086426, + 1, + 1706277632.7140539, + 1706277633.4522555, + {} + ], + [ + 15, + null, + 209652396, + null, + 0.04341844630145464, + 1.4842357635498047, + 1, + 1706277632.830031, + 1706277634.3161619, + {} + ], + [ + 12, + null, + 924231285, + null, + 0.08125657691117305, + 1.5107638835906982, + 1, + 1706277632.9116373, + 1706277634.4242318, + {} + ], + [ + 6, + null, + 924231285, + null, + 0.36005261528938415, + 3.516299247741699, + 1, + 1706277633.4184535, + 1706277636.9357994, + {} + ], + [ + 16, + null, + 209652396, + null, + 0.05231352522438881, + 1.0166974067687988, + 1, + 1706277633.539396, + 1706277634.5570924, + {} + ], + [ + 17, + null, + 209652396, + null, + 0.15859950479727636, + 0.510662317276001, + 1, + 1706277634.4225478, + 1706277634.9349742, + {} + ], + [ + 15, + null, + 398764591, + null, + 0.043413803775920856, + 1.1191012859344482, + 1, + 1706277634.5313032, + 1706277635.651467, + {} + ], + [ + 18, + null, + 209652396, + null, + 0.03673320953265247, + 1.036933183670044, + 1, + 1706277635.221057, + 1706277636.2595854, + {} + ], + [ + 16, + null, + 398764591, + null, + 0.042296502630764565, + 1.2329812049865723, + 1, + 1706277635.3001854, + 1706277636.5348752, + {} + ], + [ + 17, + null, + 398764591, + null, + 0.1229944289693593, + 0.5270049571990967, + 1, + 1706277635.3790934, + 1706277635.907605, + {} + ], + [ + 19, + null, + 209652396, + null, + 0.04006809037449699, + 1.2701239585876465, + 1, + 1706277635.5504642, + 1706277636.8224409, + {} + ], + [ + 11, + null, + 398764591, + null, + 0.0395094398019189, + 3.738813638687134, + 1, + 1706277635.738447, + 1706277639.4788716, + {} + ], + [ + 15, + null, + 924231285, + null, + 0.036725471990095904, + 1.2338676452636719, + 1, + 1706277635.8669333, + 1706277637.1028805, + {} + ], + [ + 9, + null, + 924231285, + null, + 0.03617301145156304, + 3.797776222229004, + 1, + 1706277635.9984384, + 1706277639.797911, + {} + ], + [ + 17, + null, + 924231285, + null, + 0.11129990714948934, + 0.5645942687988281, + 1, + 1706277636.3598018, + 1706277636.9253795, + {} + ], + [ + 18, + null, + 398764591, + null, + 0.040072732900030994, + 1.1205317974090576, + 1, + 1706277636.6408405, + 1706277637.7630773, + {} + ], + [ + 16, + null, + 924231285, + null, + 0.03450324976787367, + 1.0402038097381592, + 1, + 1706277636.9236588, + 1706277637.9659233, + {} + ], + [ + 19, + null, + 398764591, + null, + 0.038952336737852145, + 0.7655572891235352, + 1, + 1706277637.010183, + 1706277637.7774007, + {} + ] + ], + "configs": { + "1": { + "activation": "logistic", + "n_layer": 4, + "n_neurons": 11, + "solver": "lbfgs" + }, + "2": { + "activation": "tanh", + "n_layer": 1, + "n_neurons": 146, + "solver": "adam", + "batch_size": 77, + "learning_rate_init": 0.000534923804864797 + }, + "3": { + "activation": "relu", + "n_layer": 5, + "n_neurons": 43, + "solver": "sgd", + "batch_size": 157, + "learning_rate": "invscaling", + "learning_rate_init": 0.08903539757637764 + }, + "4": { + "activation": "logistic", + "n_layer": 2, + "n_neurons": 90, + "solver": "adam", + "batch_size": 175, + "learning_rate_init": 0.004156370184967407 + }, + "5": { + "activation": "logistic", + "n_layer": 4, + "n_neurons": 200, + "solver": "sgd", + "batch_size": 204, + "learning_rate": "constant", + "learning_rate_init": 0.016071068084706618 + }, + "6": { + "activation": "tanh", + "n_layer": 4, + "n_neurons": 63, + "solver": "sgd", + "batch_size": 144, + "learning_rate": "invscaling", + "learning_rate_init": 0.0056279320474151686 + }, + "7": { + "activation": "relu", + "n_layer": 5, + "n_neurons": 29, + "solver": "adam", + "batch_size": 173, + "learning_rate_init": 0.503948959867121 + }, + "8": { + "activation": "logistic", + "n_layer": 1, + "n_neurons": 8, + "solver": "adam", + "batch_size": 240, + "learning_rate_init": 0.8212461922256864 + }, + "9": { + "activation": "relu", + "n_layer": 3, + "n_neurons": 118, + "solver": "lbfgs" + }, + "10": { + "activation": "tanh", + "n_layer": 1, + "n_neurons": 35, + "solver": "lbfgs" + }, + "11": { + "activation": "tanh", + "n_layer": 4, + "n_neurons": 80, + "solver": "lbfgs" + }, + "12": { + "activation": "tanh", + "n_layer": 3, + "n_neurons": 27, + "solver": "lbfgs" + }, + "13": { + "activation": "logistic", + "n_layer": 5, + "n_neurons": 16, + "solver": "lbfgs" + }, + "14": { + "activation": "relu", + "n_layer": 4, + "n_neurons": 12, + "solver": "lbfgs" + }, + "15": { + "activation": "tanh", + "n_layer": 1, + "n_neurons": 63, + "solver": "lbfgs" + }, + "16": { + "activation": "relu", + "n_layer": 2, + "n_neurons": 43, + "solver": "lbfgs" + }, + "17": { + "activation": "relu", + "n_layer": 2, + "n_neurons": 18, + "solver": "lbfgs" + }, + "18": { + "activation": "tanh", + "n_layer": 1, + "n_neurons": 82, + "solver": "lbfgs" + }, + "19": { + "activation": "relu", + "n_layer": 1, + "n_neurons": 57, + "solver": "lbfgs" + } + }, + "config_origins": { + "1": "Initial Design: Sobol", + "2": "Initial Design: Sobol", + "3": "Initial Design: Sobol", + "4": "Initial Design: Sobol", + "5": "Initial Design: Sobol", + "6": null, + "7": null, + "8": null, + "9": null, + "10": "Acquisition Function Maximizer: Random Search (sorted)", + "11": "Acquisition Function Maximizer: Random Search (sorted)", + "12": "Acquisition Function Maximizer: Random Search (sorted)", + "13": "Acquisition Function Maximizer: Random Search (sorted)", + "14": "Acquisition Function Maximizer: Random Search (sorted)", + "15": "Acquisition Function Maximizer: Local Search", + "16": "Acquisition Function Maximizer: Local Search", + "17": "Acquisition Function Maximizer: Local Search", + "18": "Acquisition Function Maximizer: Random Search (sorted)", + "19": "Acquisition Function Maximizer: Random Search (sorted)" + } +} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/scenario 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/scenario 2.json new file mode 100644 index 00000000..679a91c6 --- /dev/null +++ b/logs/SMAC3v2/mlp-nondeterministic/run_2/scenario 2.json @@ -0,0 +1,155 @@ +{ + "name": "e262f9639f7cb897f1e597cb5b717d0c", + "deterministic": false, + "objectives": "cost", + "crash_cost": Infinity, + "termination_cost_threshold": Infinity, + "walltime_limit": Infinity, + "cputime_limit": Infinity, + "trial_walltime_limit": null, + "trial_memory_limit": null, + "n_trials": 50, + "use_default_config": false, + "instances": null, + "instance_features": null, + "min_budget": null, + "max_budget": null, + "seed": 0, + "n_workers": 8, + "_meta": { + "facade": { + "name": "HyperparameterOptimizationFacade" + }, + "runner": { + "name": "DaskParallelRunner" + }, + "model": { + "name": "RandomForest", + "types": [ + 3, + 0, + 0, + 3, + 0, + 4, + 0 + ], + "bounds": [ + [ + 3, + NaN + ], + [ + 0, + 1.0 + ], + [ + 0, + 1.0 + ], + [ + 3, + NaN + ], + [ + -1.0, + 1.0 + ], + [ + 4, + NaN + ], + [ + -1.0, + 1.0 + ] + ], + "pca_components": 7, + "n_trees": 10, + "n_points_per_tree": -1, + "ratio_features": 1.0, + "min_samples_split": 2, + "min_samples_leaf": 1, + "max_depth": 1048576, + "eps_purity": 1e-08, + "max_nodes": 1048576, + "bootstrapping": true + }, + "acquisition_maximizer": { + "name": "LocalAndSortedRandomSearch", + "acquisition_function": { + "name": "EI", + "xi": 0.0, + "log": true + }, + "challengers": 10000, + "seed": 0, + "random_search": { + "name": "RandomSearch", + "acquisition_function": { + "name": "EI", + "xi": 0.0, + "log": true + }, + "challengers": 5000, + "seed": 0 + }, + "local_search": { + "name": "LocalSearch", + "acquisition_function": { + "name": "EI", + "xi": 0.0, + "log": true + }, + "challengers": 5000, + "seed": 0, + "max_steps": null, + "n_steps_plateau_walk": 10, + "vectorization_min_obtain": 2, + "vectorization_max_obtain": 64 + } + }, + "acquisition_function": { + "name": "EI", + "xi": 0.0, + "log": true + }, + "intensifier": { + "name": "Intensifier", + "max_incumbents": 10, + "seed": 0 + }, + "initial_design": { + "name": "SobolInitialDesign", + "n_configs": 5, + "n_configs_per_hyperparameter": 10, + "additional_configs": [], + "seed": 0 + }, + "random_design": { + "name": "ProbabilityRandomDesign", + "seed": 0, + "probability": 0.2 + }, + "runhistory_encoder": { + "name": "RunHistoryLogScaledEncoder", + "considered_states": [ + 1, + 2, + 4 + ], + "lower_budget_states": [], + "scale_percentage": 5, + "seed": 0 + }, + "multi_objective_algorithm": null, + "config_selector": { + "name": "ConfigSelector", + "retrain_after": 8, + "retries": 16, + "min_trials": 1 + }, + "version": "2.0.2" + }, + "output_directory": "smac3_output/e262f9639f7cb897f1e597cb5b717d0c/0" +} \ No newline at end of file diff --git a/tests/test_runs/__init__ 2.py b/tests/test_runs/__init__ 2.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_utils/__init__ 2.py b/tests/test_utils/__init__ 2.py new file mode 100644 index 00000000..e69de29b From c6fd863f3e19b365efdc00cf1ad0834f822e6928 Mon Sep 17 00:00:00 2001 From: Kristina Thieme Date: Mon, 7 Oct 2024 13:42:13 +0200 Subject: [PATCH 2/6] added run validation method to each converter --- deepcave/layouts/general.py | 7 +++---- deepcave/runs/converters/amltk.py | 23 +++++++++++++++++++++++ deepcave/runs/converters/bohb.py | 21 +++++++++++++++++++++ deepcave/runs/converters/dataframe.py | 22 ++++++++++++++++++++++ deepcave/runs/converters/deepcave.py | 21 +++++++++++++++++++++ deepcave/runs/converters/optuna 2.py | 22 ++++++++++++++++++++++ deepcave/runs/converters/optuna.py | 22 ++++++++++++++++++++++ deepcave/runs/converters/smac3v1.py | 23 +++++++++++++++++++++++ deepcave/runs/converters/smac3v2.py | 23 +++++++++++++++++++++++ deepcave/runs/handler.py | 20 ++++++++++++++++++++ deepcave/runs/run.py | 19 +++++++++++++++++++ 11 files changed, 219 insertions(+), 4 deletions(-) diff --git a/deepcave/layouts/general.py b/deepcave/layouts/general.py index 4f4dd755..6088d58c 100644 --- a/deepcave/layouts/general.py +++ b/deepcave/layouts/general.py @@ -162,11 +162,11 @@ def callback(run_paths: List[str]): for i, run_path in enumerate(run_paths): run_name = run_handler.get_run_name(run_path) - is_run = run_handler.add_run(run_path) - # Differenciate between run and directory for visibility reasons + is_run = run_handler.is_run(run_path) + # Differenciate between run and directory for visibility and usability reasons if is_run: - run_handler.remove_run(run_path) + # run_handler.remove_run(run_path) new_element = html.Div( [ dbc.Button( @@ -201,7 +201,6 @@ def callback(run_paths: List[str]): ], className="mb-1", ) - print(new_element) children.append(new_element) if len(children) == 0: return html.Div("No runs found.") diff --git a/deepcave/runs/converters/amltk.py b/deepcave/runs/converters/amltk.py index 3f1becef..56017c26 100644 --- a/deepcave/runs/converters/amltk.py +++ b/deepcave/runs/converters/amltk.py @@ -24,6 +24,7 @@ from typing import Optional, Sequence, Union +import os import re from pathlib import Path @@ -215,3 +216,25 @@ def from_path(cls, path: Union[Path, str]) -> "AMLTKRun": ) return run + + @classmethod + def is_valid_run(cls, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid amltk run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + if os.path.isfile(path_name + "/history.parquet") and os.path.isfile( + path_name + "/configspace.json" + ): + return True + return False diff --git a/deepcave/runs/converters/bohb.py b/deepcave/runs/converters/bohb.py index 08b2f37c..4a39f69d 100644 --- a/deepcave/runs/converters/bohb.py +++ b/deepcave/runs/converters/bohb.py @@ -25,6 +25,7 @@ from typing import Any, Dict, Union +import os from pathlib import Path from ConfigSpace.configuration_space import ConfigurationSpace @@ -177,3 +178,23 @@ def from_path(cls, path: Union[Path, str]) -> "BOHBRun": ) return run + + @classmethod + def is_valid_run(cls, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid bohb run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + if os.path.isfile(path_name + "/configspace.json"): + return True + return False diff --git a/deepcave/runs/converters/dataframe.py b/deepcave/runs/converters/dataframe.py index 11e34048..7af0fb73 100644 --- a/deepcave/runs/converters/dataframe.py +++ b/deepcave/runs/converters/dataframe.py @@ -385,3 +385,25 @@ def _extract_additional( additional = data.drop(hyperparameters + costs_metrics + budgets + seeds + meta) additional = dict(additional) return {key: value if pd.notna(value) else None for key, value in additional.items()} + + @classmethod + def is_valid_run(cls, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid dataframe run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + if os.path.isfile(path_name + "/trials.csv") and os.path.isfile( + path_name + "/configspace.csv" + ): + return True + return False diff --git a/deepcave/runs/converters/deepcave.py b/deepcave/runs/converters/deepcave.py index 9fedc619..3e3c6324 100644 --- a/deepcave/runs/converters/deepcave.py +++ b/deepcave/runs/converters/deepcave.py @@ -24,6 +24,7 @@ from typing import Union +import os from pathlib import Path from deepcave.runs.run import Run @@ -92,3 +93,23 @@ def from_path(cls, path: Path) -> "DeepCAVERun": The DeepCAVE run. """ return DeepCAVERun(path.stem, path=Path(path)) + + @classmethod + def is_valid_run(cls, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid deepcave run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + if os.path.isfile(path_name + "history.jsonl"): + return True + return False diff --git a/deepcave/runs/converters/optuna 2.py b/deepcave/runs/converters/optuna 2.py index a6cd1497..d010f130 100644 --- a/deepcave/runs/converters/optuna 2.py +++ b/deepcave/runs/converters/optuna 2.py @@ -264,3 +264,25 @@ def from_path(cls, path: Union[Path, str]) -> "OptunaRun": ) return run + + @classmethod + def is_valid_run(cls, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid optuna2 run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + path = Path(path_name) + pickle_files = list(path.glob("*.pkl")) + if len(pickle_files) != 1: + return False + return True diff --git a/deepcave/runs/converters/optuna.py b/deepcave/runs/converters/optuna.py index 2d570744..bce2159a 100644 --- a/deepcave/runs/converters/optuna.py +++ b/deepcave/runs/converters/optuna.py @@ -285,3 +285,25 @@ def from_path(cls, path: Union[Path, str]) -> "OptunaRun": ) return run + + @classmethod + def is_valid_run(cls, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid optuna run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + path = Path(path_name) + pickle_files = list(path.glob("*.pkl")) + if len(pickle_files) != 1: + return False + return True diff --git a/deepcave/runs/converters/smac3v1.py b/deepcave/runs/converters/smac3v1.py index 21afe20c..c90142da 100644 --- a/deepcave/runs/converters/smac3v1.py +++ b/deepcave/runs/converters/smac3v1.py @@ -27,6 +27,7 @@ from typing import Optional, Union import json +import os from pathlib import Path import numpy as np @@ -217,3 +218,25 @@ def from_path(cls, path: Union[Path, str]) -> "SMAC3v1Run": ) return run + + @classmethod + def is_valid_run(cls, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid smac3v1 run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + if os.path.isfile(path_name + "/runhistory.json") and os.path.isfile( + path_name + "/configspace.json" + ): + return True + return False diff --git a/deepcave/runs/converters/smac3v2.py b/deepcave/runs/converters/smac3v2.py index 0c933526..7ccf0f25 100644 --- a/deepcave/runs/converters/smac3v2.py +++ b/deepcave/runs/converters/smac3v2.py @@ -28,6 +28,7 @@ from typing import Union import json +import os from pathlib import Path import numpy as np @@ -216,3 +217,25 @@ def from_path(cls, path: Union[Path, str]) -> "SMAC3v2Run": ) return run + + @classmethod + def is_valid_run(cls, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid smac3v2 run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + if os.path.isfile(path_name + "/runhistory.json") and os.path.isfile( + path_name + "/configspace.json" + ): + return True + return False diff --git a/deepcave/runs/handler.py b/deepcave/runs/handler.py index 27387348..6b8057c7 100644 --- a/deepcave/runs/handler.py +++ b/deepcave/runs/handler.py @@ -523,3 +523,23 @@ def get_runs(self, include_groups: bool = False) -> List[AbstractRun]: runs += list(self.groups.values()) return runs + + def is_run(self, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + for run_class in self.available_run_classes: + if run_class.is_valid_run(path_name): + return True + return False diff --git a/deepcave/runs/run.py b/deepcave/runs/run.py index 18e3475c..912db646 100644 --- a/deepcave/runs/run.py +++ b/deepcave/runs/run.py @@ -139,6 +139,25 @@ def from_path(cls, path: Path) -> "Run": """ pass + @classmethod + @abstractmethod + def is_valid_run(cls, path_name: str) -> bool: + """ + Check whether the path name belongs to a valid run. + + Parameters + ---------- + path_name: str + The path to check. + + Returns + ------- + bool + True if path is valid run. + False otherwise. + """ + pass + @property def id(self) -> str: """ From 1cf53f378c058f2c11b45e87fc0d0ee4f44961ca Mon Sep 17 00:00:00 2001 From: Kristina Thieme Date: Mon, 7 Oct 2024 14:32:57 +0200 Subject: [PATCH 3/6] added documentation, fixed minor bugs --- CHANGELOG.md | 10 ++++++++++ deepcave/layouts/general.py | 2 +- docs/converters/bohb.rst | 8 +++++++- docs/converters/custom_converters.rst | 19 +++++++++++++++---- docs/converters/deepcave.rst | 8 +++++++- docs/converters/optuna.rst | 3 +++ docs/converters/smac.rst | 11 ++++++++++- 7 files changed, 53 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a8cfbdd..59288d62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +# Version 1.3.3 + +## Converters +- Add for each converter a method to check whether a run is valid. +- Change rst docs of converters. + +## General Layout +- Fix navigation and button problems in general layout. +- Change general layout to be more intuitiv. + # Version 1.3.2 ## Features diff --git a/deepcave/layouts/general.py b/deepcave/layouts/general.py index 6088d58c..3191892b 100644 --- a/deepcave/layouts/general.py +++ b/deepcave/layouts/general.py @@ -145,6 +145,7 @@ def callback(run_paths: List[str]): html.I( className="fas fa-folder-open fa-lg", id={"type": "general-dynamic-add-run", "index": -1}, + style={"pointer-events": "none"}, ), dbc.Button( "..", @@ -166,7 +167,6 @@ def callback(run_paths: List[str]): is_run = run_handler.is_run(run_path) # Differenciate between run and directory for visibility and usability reasons if is_run: - # run_handler.remove_run(run_path) new_element = html.Div( [ dbc.Button( diff --git a/docs/converters/bohb.rst b/docs/converters/bohb.rst index 0f289929..89d1ae0b 100644 --- a/docs/converters/bohb.rst +++ b/docs/converters/bohb.rst @@ -5,4 +5,10 @@ To load an BOHB run into DeepCAVE, it is necessary to install HpBandSter, e.g. v .. code-block:: bash - pip install deepcave[bohb] \ No newline at end of file + pip install deepcave[bohb] + +To be able to load it correctly into DeepCAVE, you need save the configuration space as a ConfigSpace JSON file: + + .. code-block:: python + + "configspace.json" \ No newline at end of file diff --git a/docs/converters/custom_converters.rst b/docs/converters/custom_converters.rst index 015892ca..9f5467ff 100644 --- a/docs/converters/custom_converters.rst +++ b/docs/converters/custom_converters.rst @@ -27,23 +27,31 @@ Step 1: Create the converter class @property def hash(self) -> str: """ - Returns a unique hash for the run (e.g., hashing the trial history). + Return a unique hash for the run (e.g., hashing the trial history). """ pass @property def latest_change(self) -> float: """ - Returns the timestamp of the latest change. + Return the timestamp of the latest change. """ pass @classmethod def from_path(cls, path: str) -> 'Run': """ - Returns a Run object from a given path. + Return a Run object from a given path. """ pass + + @classmethod + def is_valid_run(cls, path: str) -> bool: + """ + Check if the path belongs to a valid run. + """ + pass + Step 2: Register your converter ------------------------------- @@ -63,7 +71,10 @@ Step 3: Implement the methods Inside this method, you have to provide a configspace, objectives, and meta data, as well as the trials, which have to be added to the newly created run object. -Step 4: Document your converter +4. **is_valid_run**: This method sould return True, if the given path belongs to a valid run. Valid means, that the given run + directory contains all files that are necessary for further processing (e.g. a configspace.json). + +Step 5: Document your converter ------------------------------- To document your converter, create a corresponding `.rst` file under `docs/converters` and add it to diff --git a/docs/converters/deepcave.rst b/docs/converters/deepcave.rst index e89ad522..09d2e27e 100644 --- a/docs/converters/deepcave.rst +++ b/docs/converters/deepcave.rst @@ -2,4 +2,10 @@ DeepCAVE --------- You can incorporate DeepCAVE into your AutoML tool directly to save your results in the native -DeepCAVE format and then analyze it in DeepCAVE. Please see the :ref:`examples` on this. \ No newline at end of file +DeepCAVE format and then analyze it in DeepCAVE. Please see the :ref:`examples` on this. + +To be able to load it correctly into DeepCAVE, your run needs to at least contain a history JSONL file: + + .. code-block:: python + + "history.jsonl" \ No newline at end of file diff --git a/docs/converters/optuna.rst b/docs/converters/optuna.rst index 89467972..932a30f5 100644 --- a/docs/converters/optuna.rst +++ b/docs/converters/optuna.rst @@ -14,6 +14,9 @@ consider when running Optuna: Replace ```` with the directory where you want to store the Pickle file. +.. note:: + There has to be exactly one Pickle file. + 2. Optional: If you want to display the names of your objectives in DeepCAVE, you can set metric names in Optuna using the following command: .. code-block:: python diff --git a/docs/converters/smac.rst b/docs/converters/smac.rst index dabb3937..e6f75eda 100644 --- a/docs/converters/smac.rst +++ b/docs/converters/smac.rst @@ -5,4 +5,13 @@ For the SMAC framework, we support loading runs created with the older SMAC vers runs created with the newer SMAC version >= 2.0.0. As Auto-Sklearn and Auto-PyTorch rely on SMAC for hyperparameter optimization, the SMAC -converter also allows to load runs created with these tools. \ No newline at end of file +converter also allows to load runs created with these tools. + +To be able to load it correctly into DeepCAVE, you need save the configuration space as a ConfigSpace JSON file, +as well as the run history: + + .. code-block:: python + + "configspace.json" + "runhistory.json" + From 6bb1eeeba1a8ec2f58e3bc314e71f95dd12264c2 Mon Sep 17 00:00:00 2001 From: Kristina Thieme Date: Tue, 8 Oct 2024 13:09:39 +0200 Subject: [PATCH 4/6] deleted weird duplicated files --- .../hyperparameter/configuration_cube 2.py | 440 --- .../hyperparameter/parallel_coordinates 2.py | 431 --- deepcave/runs/converters/optuna 2.py | 288 -- docs/plugins/symbolic_explanations 2.rst | 22 - docs/sg_execution_times.rst | 58 + examples/api/importances 2.py | 36 - logs/README 2.rst | 11 - .../run_1/configspace 2.json | 1 - .../run_1/intensifier 2.json | 145 - .../run_1/optimization 2.json | 6 - .../run_1/runhistory 2.json | 2393 ----------------- .../run_1/scenario 2.json | 155 -- .../run_2/configspace 2.json | 1 - .../run_2/intensifier 2.json | 74 - .../run_2/optimization 2.json | 6 - .../run_2/runhistory 2.json | 766 ------ .../run_2/scenario 2.json | 155 -- tests/test_runs/__init__ 2.py | 0 tests/test_utils/__init__ 2.py | 0 19 files changed, 58 insertions(+), 4930 deletions(-) delete mode 100644 deepcave/plugins/hyperparameter/configuration_cube 2.py delete mode 100644 deepcave/plugins/hyperparameter/parallel_coordinates 2.py delete mode 100644 deepcave/runs/converters/optuna 2.py delete mode 100644 docs/plugins/symbolic_explanations 2.rst create mode 100644 docs/sg_execution_times.rst delete mode 100644 examples/api/importances 2.py delete mode 100644 logs/README 2.rst delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/configspace 2.json delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/intensifier 2.json delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/optimization 2.json delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/runhistory 2.json delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_1/scenario 2.json delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/configspace 2.json delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/intensifier 2.json delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/optimization 2.json delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/runhistory 2.json delete mode 100644 logs/SMAC3v2/mlp-nondeterministic/run_2/scenario 2.json delete mode 100644 tests/test_runs/__init__ 2.py delete mode 100644 tests/test_utils/__init__ 2.py diff --git a/deepcave/plugins/hyperparameter/configuration_cube 2.py b/deepcave/plugins/hyperparameter/configuration_cube 2.py deleted file mode 100644 index ee6a4051..00000000 --- a/deepcave/plugins/hyperparameter/configuration_cube 2.py +++ /dev/null @@ -1,440 +0,0 @@ -# noqa: D400 -""" -# ConfigurationCube - -This module provides utilities for visualizing and creating a configuration cube. - -The configuration cube displays configurations and their score on the objective. - -## Classes - - ConfigurationCube: A plugin for visualizing a configuration cube. -""" - -from typing import Any, Callable, Dict, List, Tuple - -import dash_bootstrap_components as dbc -import numpy as np -import pandas as pd -import plotly.graph_objs as go -from dash import dcc, html -from dash.exceptions import PreventUpdate - -from deepcave import config -from deepcave.plugins.dynamic import DynamicPlugin -from deepcave.runs import AbstractRun, Status -from deepcave.utils.compression import deserialize, serialize -from deepcave.utils.layout import ( - get_checklist_options, - get_select_options, - get_slider_marks, - help_button, -) -from deepcave.utils.logs import get_logger -from deepcave.utils.styled_plotty import ( - get_hovertext_from_config, - get_hyperparameter_ticks, - save_image, -) - -logger = get_logger(__name__) - - -class ConfigurationCube(DynamicPlugin): - """A plugin for visualizing a configuration cube.""" - - id = "ccube" - name = "Configuration Cube" - icon = "fas fa-cube" - activate_run_selection = True - help = "docs/plugins/configuration_cube.rst" - - @staticmethod - def get_input_layout(register: Callable) -> List[dbc.Row]: - """ - Get the layout for the input block. - - Parameters - ---------- - register : Callable - Method to register (user) variables. - The register_input function is located in the Plugin superclass. - - Returns - ------- - List[dbc.Row] - Layouts for the input block. - """ - return [ - dbc.Row( - [ - dbc.Col( - [ - dbc.Label("Objective"), - dbc.Select( - id=register("objective_id", ["value", "options"], type=int), - placeholder="Select objective ...", - ), - ], - md=6, - ), - dbc.Col( - [ - dbc.Label("Budget"), - help_button( - "Budget refers to the multi-fidelity budget. " - "Combined budget means that the trial on the highest" - " evaluated budget is used. \n " - "Note: Selecting combined budget might be misleading if" - " a time objective is used. Often, higher budget take " - " longer to evaluate, which might negatively influence " - " the results." - ), - dbc.Select( - id=register("budget_id", ["value", "options"], type=int), - placeholder="Select budget ...", - ), - ], - md=6, - ), - ], - ), - ] - - @staticmethod - def get_filter_layout(register: Callable) -> List[html.Div]: - """ - Get the layout for the filter block. - - Parameters - ---------- - register : Callable - Method to register (user) variables. - The register_input function is located in the Plugin superclass. - - Returns - ------- - List[html.Div] - Layouts for the filter block. - """ - return [ - html.Div( - [ - dbc.Label("Number of Configurations"), - help_button( - "The number of configurations to show, these are ordered based on" - " the time at which they were evaluated." - ), - dcc.Slider( - id=register("n_configs", ["value", "min", "max", "marks"]), step=None - ), - ], - className="mb-3", - ), - html.Div( - [ - dbc.Label("Hyperparameters"), - help_button("Which hyperparameters to show. Maxium 3 active."), - dbc.Checklist( - id=register("hyperparameter_names", ["value", "options"]), inline=True - ), - ] - ), - ] - - def load_inputs(self) -> Dict[str, Any]: - """ - Load the content for the defined inputs in 'get_input_layout' and 'get_filter_layout'. - - This method is necessary to pre-load contents for the inputs. - So, if the plugin is called for the first time or there are no results in the cache, - the plugin gets its content from this method. - - Returns - ------- - Dict[str, Any] - The content to be filled. - """ - return { - "n_configs": {"min": 0, "max": 0, "marks": get_slider_marks(), "value": 0}, - "hyperparameter_names": {"options": get_checklist_options(), "value": []}, - } - - def load_dependency_inputs(self, run, _, inputs) -> Dict[str, Any]: # type: ignore - """ - Work like 'load_inputs' but called after inputs have changed. - - It is restricted to three Hyperparameters. - - Note - ---- - Only the changes have to be returned. - The returned dictionary will be merged with the inputs. - - Parameters - ---------- - run - The selected run. - inputs - Current content of the inputs. - - Returns - ------- - Dict[str, Any] - The dictionary with the changes. - """ - # Prepare objectives - objective_names = run.get_objective_names() - objective_ids = run.get_objective_ids() - objective_options = get_select_options(objective_names, objective_ids) - objective_value = inputs["objective_id"]["value"] - - # Prepare budgets - budgets = run.get_budgets(human=True) - budget_ids = run.get_budget_ids() - budget_options = get_select_options(budgets, budget_ids) - budget_value = inputs["budget_id"]["value"] - - # Prepare others - hp_names = list(run.configspace.keys()) - - # Get selected values - n_configs_value = inputs["n_configs"]["value"] - - # Pre-set values - if objective_value is None: - objective_value = objective_ids[0] - if budget_value is None: - budget_value = budget_ids[-1] - else: - budget_value = inputs["budget_id"]["value"] - - budget = run.get_budget(budget_value) - configs = run.get_configs(budget=budget) - if n_configs_value == 0: - n_configs_value = len(configs) - else: - if n_configs_value > len(configs): - n_configs_value = len(configs) - - # Restrict to three hyperparameters - selected_hps = inputs["hyperparameter_names"]["value"] - n_selected = len(selected_hps) - if n_selected > 3: - del selected_hps[0] - - return { - "objective_id": { - "options": objective_options, - "value": objective_value, - }, - "budget_id": { - "options": budget_options, - "value": budget_value, - }, - "n_configs": { - "min": 0, - "max": len(configs), - "marks": get_slider_marks(list(range(0, len(configs) + 1))), - "value": n_configs_value, - }, - "hyperparameter_names": { - "options": get_select_options(hp_names), - "value": selected_hps, - }, - } - - @staticmethod - def process(run: AbstractRun, inputs: Dict[str, Any]) -> Dict[str, str]: - """ - Return raw data based on the run and input data. - - Warning - ------- - The returned data must be JSON serializable. - - Note - ---- - The passed inputs are cleaned and therefore differs compared to 'load_inputs' - or 'load_dependency_inputs'. - Please see '_clean_inputs' for more information. - - Parameters - ---------- - run : AbstractRun - The selected run. - inputs : Dict[str, Any] - The input data. - - Returns - ------- - Dict[str, str] - The serialized dictionary. - """ - budget = run.get_budget(inputs["budget_id"]) - objective = run.get_objective(inputs["objective_id"]) - - df = run.get_encoded_data( - objectives=objective, budget=budget, statuses=Status.SUCCESS, include_config_ids=True - ) - return {"df": serialize(df)} - - @staticmethod - def get_output_layout(register: Callable) -> Tuple[dcc.Graph,]: - """ - Get the layout for the output block. - - Parameters - ---------- - register : Callable - Method to register output. - The register_output function is located in the Plugin superclass. - - Returns - ------- - Tuple[dcc.Graph,] - Layout for the output block. - """ - return ( - dcc.Graph( - register("graph", "figure"), - style={"height": config.FIGURE_HEIGHT}, - config={"toImageButtonOptions": {"scale": config.FIGURE_DOWNLOAD_SCALE}}, - ), - ) - - @staticmethod - def load_outputs(run, inputs, outputs) -> go.Figure: # type: ignore - """ - Read in the raw data and prepares them for the layout. - - Note - ---- - The passed inputs are cleaned and therefore differs compared to 'load_inputs' - or 'load_dependency_inputs'. - Please see '_clean_inputs' for more information. - - Parameters - ---------- - run - The selected run. - inputs - Input and filter values from the user. - outputs - Raw output from the run. - - Returns - ------- - go.Figure - The output figure. - """ - df = deserialize(outputs["df"], dtype=pd.DataFrame) - hp_names = inputs["hyperparameter_names"] - n_configs = inputs["n_configs"] - objective_id = inputs["objective_id"] - objective = run.get_objective(objective_id) - budget = run.get_budget(inputs["budget_id"]) - df = df.groupby(df.columns.drop(objective.name).to_list(), as_index=False).mean() - df.index = df.index.astype("str") - - # Limit to n_configs - idx = [str(i) for i in range(n_configs, len(df))] - df = df.drop(idx) - - costs = df[objective.name].values.tolist() - config_ids = df["config_id"].values.tolist() - data = [] - - # Specify layout kwargs - layout_kwargs = {} - if n_configs > 0 and len(hp_names) > 0: - for i, (hp_name, axis_name) in enumerate(zip(hp_names, ["xaxis", "yaxis", "zaxis"])): - hp = run.configspace[hp_name] - values = df[hp_name].values.tolist() - - tickvals, ticktext = get_hyperparameter_ticks(hp, ticks=4, include_nan=True) - layout_kwargs[axis_name] = { - "tickvals": tickvals, - "ticktext": ticktext, - "title": hp_name, - } - data.append(values) - - if len(data) == 0: - raise PreventUpdate - - # Transforms data to values - values = np.transpose(np.array(data)).tolist() - - # Now the duplicates are filtered - filtered_data: Dict[Tuple[int, float], Tuple] = {} - for config_id, cost, v in zip(config_ids, costs, values): - v = tuple(v) # Make hashable - key = (config_id, cost) - - if v in filtered_data.values(): - old_key = list(filtered_data.keys())[list(filtered_data.values()).index(v)] - old_cost = old_key[1] - - if objective.optimize == "lower": - if old_cost < cost: - continue - - if objective.optimize == "upper": - if old_cost > cost: - continue - - # Otherwise it is replaced - del filtered_data[old_key] - - filtered_data[key] = v - - # Replace data - config_ids = [k[0] for k in filtered_data.keys()] - costs = [k[1] for k in filtered_data.keys()] - - # Specify scatter kwargs - scatter_kwargs = { - "mode": "markers", - "marker": { - "size": 5, - "color": costs, - "colorbar": {"thickness": 30, "title": objective.name}, - }, - "hovertext": [ - get_hovertext_from_config(run, config_id, budget) for config_id in config_ids - ], - "meta": {"colorbar": costs}, - "hoverinfo": "text", - } - - if len(data) >= 1: - x = [v[0] for v in filtered_data.values()] - if len(data) >= 2: - y = [v[1] for v in filtered_data.values()] - if len(data) >= 3: - z = [v[2] for v in filtered_data.values()] - - if len(data) == 3: - trace = go.Scatter3d(x=x, y=y, z=z, **scatter_kwargs) - layout = go.Layout( - {"scene": {**layout_kwargs}}, - font=dict(size=config.FIGURE_FONT_SIZE), - ) - else: - if len(data) == 1: - y = [0 for _ in x] - trace = go.Scatter(x=x, y=y, **scatter_kwargs) - elif len(data) == 2: - trace = go.Scatter(x=x, y=y, **scatter_kwargs) - else: - trace = go.Scatter(x=[], y=[]) - layout = go.Layout( - **layout_kwargs, - font=dict(size=config.FIGURE_FONT_SIZE), - ) - - figure = go.Figure(data=trace, layout=layout) - figure.update_layout(dict(margin=config.FIGURE_MARGIN)) - save_image(figure, "configuration_cube.pdf") - - return figure diff --git a/deepcave/plugins/hyperparameter/parallel_coordinates 2.py b/deepcave/plugins/hyperparameter/parallel_coordinates 2.py deleted file mode 100644 index b2b0bb84..00000000 --- a/deepcave/plugins/hyperparameter/parallel_coordinates 2.py +++ /dev/null @@ -1,431 +0,0 @@ -# noqa: D400 - -""" -# ParallelCoordinates - -This module provides utilities for visualizing the parallel coordinates. - -## Classes - - ParallelCoordinates : Can be used for visualizing the parallel coordinates. -""" - -from typing import Any, Callable, Dict, List - -from collections import defaultdict - -import dash_bootstrap_components as dbc -import numpy as np -import pandas as pd -import plotly.graph_objs as go -from dash import dcc, html -from dash.exceptions import PreventUpdate - -from deepcave import config -from deepcave.constants import VALUE_RANGE -from deepcave.evaluators.fanova import fANOVA -from deepcave.plugins.static import StaticPlugin -from deepcave.utils.compression import deserialize, serialize -from deepcave.utils.layout import get_checklist_options, get_select_options, help_button -from deepcave.utils.logs import get_logger -from deepcave.utils.styled_plotty import get_hyperparameter_ticks, save_image - -logger = get_logger(__name__) - - -class ParallelCoordinates(StaticPlugin): - """Can be used for visualizing the parallel coordinates.""" - - id = "parallel_coordinates" - name = "Parallel Coordinates" - icon = "far fa-map" - activate_run_selection = True - help = "docs/plugins/parallel_coordinates.rst" - - @staticmethod - def get_input_layout(register: Callable) -> List[Any]: - """ - Get the layout for the input block. - - Parameters - ---------- - register : Callable - Method to regsiter (user) variables. - The register_input function is located in the Plugin superclass. - - Returns - ------- - List[Any] - The layouts for the input block. - """ - return [ - dbc.Row( - [ - dbc.Col( - [ - dbc.Label("Objective"), - dbc.Select( - id=register("objective_id", ["value", "options"], type=int), - placeholder="Select objective ...", - ), - ], - md=6, - ), - dbc.Col( - [ - dbc.Label("Budget"), - help_button( - "Budget refers to the multi-fidelity budget. " - "Combined budget means that the trial on the highest evaluated" - " budget is used. \n " - "Note: Selecting combined budget might be misleading if a time" - " objective is used. Often, higher budget take longer to evaluate," - " which might negatively influence the results." - ), - dbc.Select( - id=register("budget_id", ["value", "options"], type=int), - placeholder="Select budget ...", - ), - ], - md=6, - ), - ], - className="mb-3", - ), - html.Div( - [ - dbc.Label("Show Important Hyperparameters"), - help_button( - "Only the most important hyperparameters are shown which are " - "calculated by fANOVA using 10 trees. The more left a " - "hyperparameter stands, the more important it is. However, activating " - "this option might take longer." - ), - dbc.Select( - id=register("show_important_only", ["value", "options"]), - placeholder="Select ...", - ), - ] - ), - ] - - @staticmethod - def get_filter_layout(register: Callable) -> List[Any]: - """ - Get the layout for the filter block. - - Parameters - ---------- - register : Callable - Method to register (user) variables. - The register_input function is located in the Plugin superclass. - - Returns - ------- - List[Any] - The layouts for the filter block. - """ - return [ - dbc.Row( - [ - dbc.Col( - [ - dbc.Label("Limit Hyperparameters"), - help_button( - "Shows either the n most important hyperparameters (if show " - "important hyperparameters is true) or the first n selected " - "hyperparameters." - ), - dbc.Input(id=register("n_hps", "value"), type="number"), - ], - md=6, - ), - dbc.Col( - [ - dbc.Label("Show Unsuccessful Configurations"), - help_button("Whether to show all configurations or only failed ones."), - dbc.Select( - id=register("show_unsuccessful", ["value", "options"]), - placeholder="Select ...", - ), - ], - md=6, - ), - ], - ), - html.Div( - [ - dbc.Label("Hyperparameters"), - dbc.Checklist( - id=register("hyperparameter_names", ["value", "options"]), inline=True - ), - ], - className="mt-3", - id=register("hide_hps", ["hidden"]), - ), - ] - - def load_inputs(self) -> Dict[str, Dict[str, Any]]: - """ - Load the content for the defined inputs in 'get_input_layout' and 'get_filter_layout'. - - This method is necessary to pre-load contents for the inputs. - So, if the plugin is called for the first time or there are no results in the cache, - the plugin gets its content from this method. - - Returns - ------- - Dict[str, Dict[str, Any]] - Content to be filled. - """ - return { - "show_important_only": {"options": get_select_options(binary=True), "value": "true"}, - "show_unsuccessful": {"options": get_select_options(binary=True), "value": "false"}, - "n_hps": {"value": 0}, - "hyperparameter_names": {"options": get_checklist_options(), "value": []}, - "hide_hps": {"hidden": True}, - } - - def load_dependency_inputs(self, run, _, inputs) -> Dict[str, Any]: # type: ignore - """ - Work like 'load_inputs' but called after inputs have changed. - - Note - ---- - Only the changes have to be returned. - The returned dictionary will be merged with the inputs. - - Parameters - ---------- - run - The selected run. - inputs - Current content of the inputs. - - Returns - ------- - Dict[str, Any] - The dictionary with the changes. - """ - # Prepare objectives - objective_names = run.get_objective_names() - objective_ids = run.get_objective_ids() - objective_options = get_select_options(objective_names, objective_ids) - objective_value = inputs["objective_id"]["value"] - - # Prepare budgets - budgets = run.get_budgets(human=True) - budget_ids = run.get_budget_ids() - budget_options = get_checklist_options(budgets, budget_ids) - budget_value = inputs["budget_id"]["value"] - - # Prepare others - n_hps = inputs["n_hps"]["value"] - hp_names = list(run.configspace.keys()) - - if inputs["show_important_only"]["value"] == "true": - hp_options = [] - hp_value = inputs["hyperparameter_names"]["value"] - hidden = True - else: - hp_options = get_select_options(hp_names) - - values = inputs["hyperparameter_names"]["value"] - if len(values) == 0: - values = hp_names - - hp_value = values - hidden = False - - if objective_value is None: - objective_value = objective_ids[0] - budget_value = budget_ids[-1] - hp_value = hp_names - - if n_hps == 0: - n_hps = len(hp_names) - - return { - "objective_id": { - "options": objective_options, - "value": objective_value, - }, - "budget_id": { - "options": budget_options, - "value": budget_value, - }, - "hyperparameter_names": { - "options": hp_options, - "value": hp_value, - }, - "n_hps": {"value": n_hps}, - "hide_hps": {"hidden": hidden}, - } - - @staticmethod - def process(run, inputs) -> Dict[str, Any]: # type: ignore - """ - Return raw data based on a run and input data. - - Warning - ------- - The returned data must be JSON serializable. - - Note - ---- - The passed inputs are cleaned and therefore differs compared to 'load_inputs' - or 'load_dependency_inputs'. - Please see '_clean_inputs' for more information. - - Parameters - ---------- - run : AbstractRun - The run to process. - inputs : Dict[str, Any] - The input data. - - Returns - ------- - Dict[str, Any] - The serialized dictionary. - """ - budget = run.get_budget(inputs["budget_id"]) - objective = run.get_objective(inputs["objective_id"]) - df = run.get_encoded_data(objective, budget) - df = df.groupby(df.columns.drop(objective.name).to_list(), as_index=False).mean() - df = serialize(df) - result: Dict[str, Any] = {"df": df} - - if inputs["show_important_only"]: - # Let's run a quick fANOVA here - evaluator = fANOVA(run) - evaluator.calculate(objective, budget, n_trees=10, seed=0) - importances_dict = evaluator.get_importances() - importances = {u: v[0] for u, v in importances_dict.items()} - important_hp_names = sorted( - importances, key=lambda key: importances[key], reverse=False - ) - result["important_hp_names"] = important_hp_names - - return result - - @staticmethod - def get_output_layout(register: Callable) -> dcc.Graph: - """ - Get the layout for the output block. - - Parameters - ---------- - register : Callable - Method to register outputs. - The register_output function is located in the Plugin superclass. - - Returns - ------- - dcc.Graph - The layouts for the output block. - """ - return dcc.Graph( - register("graph", "figure"), - style={"height": config.FIGURE_HEIGHT}, - config={"toImageButtonOptions": {"scale": config.FIGURE_DOWNLOAD_SCALE}}, - ) - - @staticmethod - def load_outputs(run, inputs, outputs) -> go.Figure: # type: ignore - """ - Read in the raw data and prepare them for the layout. - - Note - ---- - The passed inputs are cleaned and therefore differs compared to 'load_inputs' - or 'load_dependency_inputs'. - Please see '_clean_inputs' for more information. - - Parameters - ---------- - run - The selected run. - inputs - The inputs and filter values fromt the user. - outputs - Raw output from the run. - - Returns - ------- - go.Figure - The output figure. - """ - objective = run.get_objective(inputs["objective_id"]) - objective_name = objective.name - - show_important_only = inputs["show_important_only"] - show_unsuccessful = inputs["show_unsuccessful"] - n_hps = inputs["n_hps"] - - if n_hps == "" or n_hps is None: - raise PreventUpdate - else: - n_hps = int(n_hps) - - if show_important_only: - hp_names = outputs["important_hp_names"] - else: - hp_names = inputs["hyperparameter_names"] - - hp_names = hp_names[:n_hps] - - df = outputs["df"] - df = deserialize(df, dtype=pd.DataFrame) - objective_values = [] - for value in df[objective_name].values: - b = np.isnan(value) - if not show_unsuccessful: - b = not b - if b: - objective_values += [value] - - data: defaultdict = defaultdict(dict) - for hp_name in hp_names: - values = [] - for hp_v, objective_v in zip(df[hp_name].values, df[objective_name].values): - b = np.isnan(objective_v) - if not show_unsuccessful: - b = not b - if b: - values += [hp_v] - - data[hp_name]["values"] = values - data[hp_name]["label"] = hp_name - data[hp_name]["range"] = VALUE_RANGE - - hp = run.configspace[hp_name] - tickvals, ticktext = get_hyperparameter_ticks(hp, ticks=4, include_nan=True) - - data[hp_name]["tickvals"] = tickvals - data[hp_name]["ticktext"] = ticktext - - if show_unsuccessful: - line = dict() - else: - data[objective_name]["values"] = objective_values - data[objective_name]["label"] = objective_name - line = dict( - color=data[objective_name]["values"], - showscale=True, - colorscale="aggrnyl", - ) - - figure = go.Figure( - data=go.Parcoords( - line=line, - dimensions=list([d for d in data.values()]), - labelangle=45, - ), - layout=dict( - margin=dict(t=150, b=50, l=100, r=0), - font=dict(size=config.FIGURE_FONT_SIZE), - ), - ) - save_image(figure, "parallel_coordinates.pdf") - - return figure diff --git a/deepcave/runs/converters/optuna 2.py b/deepcave/runs/converters/optuna 2.py deleted file mode 100644 index d010f130..00000000 --- a/deepcave/runs/converters/optuna 2.py +++ /dev/null @@ -1,288 +0,0 @@ -# noqa: D400 -""" -# OptunaRun - -This module provides utilities to create an Optuna run. - -## Classes - - OptunaRun: Define an Optuna run object. -""" - -from typing import List, Union - -import pickle -from pathlib import Path - -from ConfigSpace import ( - Categorical, - Configuration, - ConfigurationSpace, - Float, - Integer, - Uniform, -) -from ConfigSpace.hyperparameters import Hyperparameter -from optuna.distributions import ( - CategoricalDistribution, - FloatDistribution, - IntDistribution, -) -from optuna.search_space import IntersectionSearchSpace -from optuna.study import StudyDirection -from optuna.trial import TrialState - -from deepcave.runs import Status -from deepcave.runs.objective import Objective -from deepcave.runs.run import Run -from deepcave.utils.hash import file_to_hash -from deepcave.utils.logs import get_logger - -logger = get_logger(__name__) - - -class OptunaRun(Run): - """ - Define an Optuna run object. - - Properties - ---------- - path : Path - The path to the run. - """ - - prefix = "Optuna" - _initial_order = 2 - - @staticmethod - def _get_pickle_file(path: Path) -> Path: - """ - Get the path to the pickle file from the directory path. - - Parameters - ---------- - path : Path - The path to the directory containing the pickle file. - - Returns - ------- - Path - The path to the pickle file. - """ - pickle_files = list(path.glob("*.pkl")) - if len(pickle_files) != 1: - raise RuntimeError("There should be exactly one pickle file in the directory.") - else: - return pickle_files[0] - - @property - def hash(self) -> str: - """ - Hash of the current run. - - If the hash changes, the cache has to be cleared. - This ensures that the cache always holds the latest results of the run. - - Returns - ------- - str - The hash of the run. - """ - if self.path is None: - return "" - - pickle_file = OptunaRun._get_pickle_file(self.path) - return file_to_hash(pickle_file) - - @property - def latest_change(self) -> Union[float, int]: - """ - Get the timestamp of the latest change. - - Returns - ------- - Union[float, int] - The latest change. - """ - if self.path is None: - return 0 - - pickle_file = OptunaRun._get_pickle_file(self.path) - return Path(pickle_file).stat().st_mtime - - @classmethod - def from_path(cls, path: Union[Path, str]) -> "OptunaRun": - """ - Based on working_dir/run_name/*, return a new trials object. - - Parameters - ---------- - path : Union[Path, str] - The path to base the trial object on. - - Returns - ------- - The Optuna run. - - Raises - ------ - RuntimeError - Instances are not supported. - """ - path = Path(path) - - # Load the optuna study from the file path - pickle_file_path = OptunaRun._get_pickle_file(path) - with open(pickle_file_path, "rb") as f: - optuna_study = pickle.load(f) - - # Read configspace - optuna_space = IntersectionSearchSpace(include_pruned=True).calculate(study=optuna_study) - configspace = ConfigurationSpace() - - hyperparameters: List[Hyperparameter] = [] - - for hp_name, hp in optuna_space.items(): - if isinstance(hp, FloatDistribution) or isinstance(hp, IntDistribution): - if hp.step is not None and hp.step != 1: - logger.warning( - f"Step is not supported. " - f'Step={hp.step} will be ignored for hyperparameter "{hp_name}".' - ) - - if isinstance(hp, FloatDistribution): - hyperparameters.append( - Float( - name=hp_name, - bounds=(hp.low, hp.high), - distribution=Uniform(), - default=None, - log=hp.log, - ) - ) - elif isinstance(hp, IntDistribution): - hyperparameters.append( - Integer( - name=hp_name, - bounds=(hp.low, hp.high), - distribution=Uniform(), - default=None, - log=hp.log, - ) - ) - elif isinstance(hp, CategoricalDistribution): - hyperparameters.append( - Categorical( - name=hp_name, - default=hp.choices[0], - items=hp.choices, - ) - ) - else: - raise ValueError( - ( - "The hyperparameters in the Optuna study must be of type " - "`FloatDistribution`, `IntDistribution` or `CategoricalDistribution`, " - f"but a hyperparameter of type {type(hp)} was given." - ) - ) - configspace.add(hyperparameters) - - n_objectives = max(len(trial.values) for trial in optuna_study.trials) - obj_list = list() - for i in range(n_objectives): - if optuna_study.metric_names is not None: - metric_name = optuna_study.metric_names[i] - else: - metric_name = f"Objective{i}" - optimize = "lower" if optuna_study.directions[i] == StudyDirection.MINIMIZE else "upper" - - obj_list.append( - Objective( - name=metric_name, - lower=None, - upper=None, - optimize=optimize, - ) - ) - - obj_list.append(Objective("Time")) - - # Let's create a new run object - run = OptunaRun(name=path.stem, configspace=configspace, objectives=obj_list, meta=None) - - # The path has to be set manually - run._path = path - - first_starttime = None - for trial in optuna_study.trials: - try: - config = Configuration(configspace, trial.params) - except ValueError: - raise ValueError( - f"Could not convert the configuration of trial {trial.number} to " - f"a ConfigSpace configuration.\nThis might be due to the " - f"configuration space containing conditionals or dynamic " - f"hyperparameter value ranges, which are currently not supported." - ) - - if first_starttime is None: - first_starttime = trial.datetime_start.timestamp() - - starttime = trial.datetime_start.timestamp() - first_starttime - endtime = trial.datetime_complete.timestamp() - first_starttime - - if trial.state == TrialState.COMPLETE: - status = Status.SUCCESS - elif trial.state == TrialState.FAIL: - status = Status.FAILED - elif trial.state == TrialState.PRUNED: - status = Status.PRUNED - elif trial.state == TrialState.RUNNING or trial["status"] == TrialState.WAITING: - continue - else: - status = Status.UNKNOWN - - cost = trial.values - - if status != Status.SUCCESS: - # Costs which failed, should not be included - cost = [None] * len(cost) if isinstance(cost, list) else None - time = None - else: - time = float(endtime - starttime) - - run.add( - costs=cost + [time] if isinstance(cost, list) else [cost, time], # type: ignore - config=config, - budget=0.0, - seed=-1, - start_time=starttime, - end_time=endtime, - status=status, - origin=None, - additional=None, - ) - - return run - - @classmethod - def is_valid_run(cls, path_name: str) -> bool: - """ - Check whether the path name belongs to a valid optuna2 run. - - Parameters - ---------- - path_name: str - The path to check. - - Returns - ------- - bool - True if path is valid run. - False otherwise. - """ - path = Path(path_name) - pickle_files = list(path.glob("*.pkl")) - if len(pickle_files) != 1: - return False - return True diff --git a/docs/plugins/symbolic_explanations 2.rst b/docs/plugins/symbolic_explanations 2.rst deleted file mode 100644 index 1b78e02b..00000000 --- a/docs/plugins/symbolic_explanations 2.rst +++ /dev/null @@ -1,22 +0,0 @@ -Symbolic Explanations -==================== - -Symbolic Explanations allow to obtain explicit formulas quantifying the relation between -hyperparameter values and model performance by applying symbolic regression to meta-data collected -during hyperparameter optimization. - -The plugin is capable of answering similar questions as the Partial Dependencies plugin, i.e.: - -* How does the objective change with respect to one or two hyperparameters? For example, does the - accuracy increase if the learning rate decreases? -* Do multiple trials show similar behavior? - -While the Partial Dependencies plugin provides a plot describing the effects of hyperparameters on -the model performance, the Symbolic Explanations plugin additionally allows to obtain an explicit -formula capturing these effects. - -To learn more about Symbolic Explanations, please see the paper -`Symbolic Explanations for Hyperparameter Optimization -`_. - -.. image:: ../images/plugins/symbolic_explanations.png diff --git a/docs/sg_execution_times.rst b/docs/sg_execution_times.rst new file mode 100644 index 00000000..1654b0ec --- /dev/null +++ b/docs/sg_execution_times.rst @@ -0,0 +1,58 @@ + +:orphan: + +.. _sphx_glr_sg_execution_times: + + +Computation times +================= +**00:00.000** total execution time for 8 files **from all galleries**: + +.. container:: + + .. raw:: html + + + + + + + + .. list-table:: + :header-rows: 1 + :class: table table-striped sg-datatable + + * - Example + - Time + - Mem (MB) + * - :ref:`sphx_glr_examples_api_ablation_paths.py` (``../examples/api/ablation_paths.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_examples_api_importances 2.py` (``../examples/api/importances 2.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_examples_api_importances.py` (``../examples/api/importances.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_examples_api_parallel_coordinates.py` (``../examples/api/parallel_coordinates.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_examples_api_pdp.py` (``../examples/api/pdp.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_examples_record_digits_sklearn.py` (``../examples/record/digits_sklearn.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_examples_record_minimal.py` (``../examples/record/minimal.py``) + - 00:00.000 + - 0.0 + * - :ref:`sphx_glr_examples_record_mnist_pytorch.py` (``../examples/record/mnist_pytorch.py``) + - 00:00.000 + - 0.0 diff --git a/examples/api/importances 2.py b/examples/api/importances 2.py deleted file mode 100644 index de4dad38..00000000 --- a/examples/api/importances 2.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Importances -^^^^^^^^^^^^^^^^^^^^ - -This example shows how to use the plugin Importances. -Note that other plugins use the same interfaces and can be used in the same fashion. -""" - -from deepcave.plugins.hyperparameter.importances import Importances -from deepcave.runs.converters.deepcave import DeepCAVERun -from pathlib import Path - - -if __name__ == "__main__": - # Instantiate the run - run = DeepCAVERun.from_path(Path("logs/DeepCAVE/minimal/run_2")) - - objective_id = run.get_objective_ids()[0] - budget_ids = run.get_budget_ids() - - # Instantiate the plugin - plugin = Importances() - inputs = plugin.generate_inputs( - hyperparameter_names=list(run.configspace.keys()), - objective_id=objective_id, - budget_ids=budget_ids, - method="global", - n_hps=3, - n_trees=10, - ) - # Note: Filter variables are not considered. - outputs = plugin.generate_outputs(run, inputs) - - # Finally, you can load the figure. Here, the filter variables play a role. - figure = plugin.load_outputs(run, inputs, outputs) - figure.write_image("examples/api/importances.png", scale=2.0) diff --git a/logs/README 2.rst b/logs/README 2.rst deleted file mode 100644 index 2cadd357..00000000 --- a/logs/README 2.rst +++ /dev/null @@ -1,11 +0,0 @@ -Example Logs -============ - -This directory contains exemplary run logs from hyperparameter optimization runs created with -AMLTK, BOHB, Optuna, and SMAC3. -Furthermore, it contains example logs created with the DeepCAVE recorder and example logs in -CSV format. - -After starting DeepCAVE, you can load these logs by navigating to the directory containing the run -log you would like to load on the `General` site and then selecting the run log via clicking on -the `+` next to it. \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/configspace 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/configspace 2.json deleted file mode 100644 index be8aa231..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_1/configspace 2.json +++ /dev/null @@ -1 +0,0 @@ -{"name": null, "hyperparameters": [{"type": "categorical", "name": "activation", "choices": ["logistic", "tanh", "relu"], "weights": null, "default_value": "tanh", "meta": null}, {"type": "uniform_int", "name": "n_layer", "lower": 1, "upper": 5, "default_value": 1, "log": false, "meta": null}, {"type": "uniform_int", "name": "n_neurons", "lower": 8, "upper": 256, "default_value": 10, "log": true, "meta": null}, {"type": "categorical", "name": "solver", "choices": ["lbfgs", "sgd", "adam"], "weights": null, "default_value": "adam", "meta": null}, {"type": "uniform_int", "name": "batch_size", "lower": 30, "upper": 300, "default_value": 200, "log": false, "meta": null}, {"type": "categorical", "name": "learning_rate", "choices": ["constant", "invscaling", "adaptive"], "weights": null, "default_value": "constant", "meta": null}, {"type": "uniform_float", "name": "learning_rate_init", "lower": 0.0001, "upper": 1.0, "default_value": 0.001, "log": true, "meta": null}], "conditions": [{"type": "IN", "child": "batch_size", "parent": "solver", "values": ["sgd", "adam"]}, {"type": "EQ", "child": "learning_rate", "parent": "solver", "value": "sgd"}, {"type": "IN", "child": "learning_rate_init", "parent": "solver", "values": ["sgd", "adam"]}], "forbiddens": [], "python_module_version": "1.2.0", "format_version": 0.4} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/intensifier 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/intensifier 2.json deleted file mode 100644 index d7b218ec..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_1/intensifier 2.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "incumbent_ids": [ - 20 - ], - "rejected_config_ids": [ - 1, - 6, - 8, - 3, - 10, - 4, - 12, - 13, - 7, - 2, - 15, - 17, - 18, - 19, - 14, - 24, - 21, - 9, - 22, - 16, - 26, - 29, - 28, - 31, - 32, - 34, - 27, - 36, - 37, - 35, - 23, - 11, - 38, - 39, - 43, - 5, - 41, - 40, - 42, - 45, - 46, - 48, - 30, - 50, - 52, - 53, - 55, - 54, - 44, - 57, - 56, - 25, - 49 - ], - "incumbents_changed": 5, - "trajectory": [ - { - "config_ids": [ - 1 - ], - "costs": [ - 0.6699473847106159 - ], - "trial": 1, - "walltime": 0.4147176742553711 - }, - { - "config_ids": [ - 8 - ], - "costs": [ - 0.12855926957598263 - ], - "trial": 2, - "walltime": 0.7400827407836914 - }, - { - "config_ids": [ - 2 - ], - "costs": [ - 0.025597338285360594 - ], - "trial": 5, - "walltime": 1.215728759765625 - }, - { - "config_ids": [ - 14 - ], - "costs": [ - 0.02392757660167144 - ], - "trial": 13, - "walltime": 2.366025686264038 - }, - { - "config_ids": [ - 20 - ], - "costs": [ - 0.02115031466006397 - ], - "trial": 37, - "walltime": 8.94394326210022 - } - ], - "state": { - "queue": [ - [ - 33, - 4 - ], - [ - 51, - 4 - ], - [ - 58, - 2 - ], - [ - 47, - 4 - ], - [ - 59, - 2 - ], - [ - 60, - 2 - ], - [ - 61, - 2 - ] - ] - } -} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/optimization 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/optimization 2.json deleted file mode 100644 index cd5ebb87..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_1/optimization 2.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "used_walltime": 67.33127307891846, - "used_target_function_walltime": 494.0225176811218, - "last_update": 1712565076.753763, - "finished": true -} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/runhistory 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/runhistory 2.json deleted file mode 100644 index 5ab1ec0f..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_1/runhistory 2.json +++ /dev/null @@ -1,2393 +0,0 @@ -{ - "stats": { - "submitted": 154, - "finished": 154, - "running": 0 - }, - "data": [ - [ - 1, - null, - 209652396, - null, - 0.6699473847106159, - 0.32405996322631836, - 1, - 1712565009.4599066, - 1712565009.7847676, - {} - ], - [ - 2, - null, - 209652396, - null, - 0.025597338285360594, - 1.0813992023468018, - 1, - 1712565009.4976892, - 1712565010.5797143, - {} - ], - [ - 3, - null, - 209652396, - null, - 0.6583333333333333, - 1.1021058559417725, - 1, - 1712565009.5342155, - 1712565010.6368752, - {} - ], - [ - 4, - null, - 209652396, - null, - 0.02560352831940571, - 1.2457053661346436, - 1, - 1712565009.573759, - 1712565010.8200808, - {} - ], - [ - 5, - null, - 209652396, - null, - 0.8992773135252243, - 10.674755573272705, - 1, - 1712565009.611309, - 1712565020.286695, - {} - ], - [ - 6, - null, - 209652396, - null, - 0.19250851129681212, - 0.5179076194763184, - 1, - 1712565009.6513555, - 1712565010.1701534, - {} - ], - [ - 7, - null, - 209652396, - null, - 0.89983286908078, - 1.9669127464294434, - 1, - 1712565009.704494, - 1712565011.67224, - {} - ], - [ - 8, - null, - 209652396, - null, - 0.12855926957598263, - 0.35997724533081055, - 1, - 1712565009.744616, - 1712565010.1052234, - {} - ], - [ - 9, - null, - 209652396, - null, - 0.05064376354069944, - 2.6614692211151123, - 1, - 1712565009.8414927, - 1712565012.5038698, - {} - ], - [ - 1, - null, - 398764591, - null, - 0.6861915815536985, - 0.3299448490142822, - 1, - 1712565010.1649523, - 1712565010.4956682, - {} - ], - [ - 10, - null, - 209652396, - null, - 0.2904967502321263, - 0.49783873558044434, - 1, - 1712565010.2330816, - 1712565010.7317798, - {} - ], - [ - 8, - null, - 398764591, - null, - 0.09072887650882078, - 0.3993697166442871, - 1, - 1712565010.5594473, - 1712565010.959439, - {} - ], - [ - 11, - null, - 209652396, - null, - 0.030058805323429305, - 8.363003253936768, - 1, - 1712565010.6421216, - 1712565019.005703, - {} - ], - [ - 2, - null, - 398764591, - null, - 0.03006035283194064, - 1.5795397758483887, - 1, - 1712565010.7179625, - 1712565012.2989554, - {} - ], - [ - 12, - null, - 209652396, - null, - 0.4312689569792634, - 0.41001152992248535, - 1, - 1712565010.7946854, - 1712565011.2055905, - {} - ], - [ - 13, - null, - 209652396, - null, - 0.8987186629526462, - 0.787147045135498, - 1, - 1712565010.8761115, - 1712565011.6642888, - {} - ], - [ - 14, - null, - 209652396, - null, - 0.02392757660167144, - 0.7371306419372559, - 1, - 1712565011.0186121, - 1712565011.7567017, - {} - ], - [ - 15, - null, - 209652396, - null, - 0.27155834107087595, - 0.5496902465820312, - 1, - 1712565011.2593076, - 1712565011.8098226, - {} - ], - [ - 16, - null, - 209652396, - null, - 0.027825750541627947, - 2.965526819229126, - 1, - 1712565011.714806, - 1712565014.681264, - {} - ], - [ - 17, - null, - 209652396, - null, - 0.2031012070566388, - 0.9088141918182373, - 1, - 1712565011.782877, - 1712565012.692426, - {} - ], - [ - 14, - null, - 398764591, - null, - 0.03616527390900648, - 0.5853779315948486, - 1, - 1712565011.84823, - 1712565012.43451, - {} - ], - [ - 18, - null, - 209652396, - null, - 0.043975549365521616, - 1.1089684963226318, - 1, - 1712565011.9250941, - 1712565013.034664, - {} - ], - [ - 19, - null, - 209652396, - null, - 0.0473026926648098, - 0.8433613777160645, - 1, - 1712565012.3523204, - 1712565013.196589, - {} - ], - [ - 20, - null, - 209652396, - null, - 0.019481584648715633, - 1.379931926727295, - 1, - 1712565012.4961815, - 1712565013.877354, - {} - ], - [ - 14, - null, - 924231285, - null, - 0.034498607242339774, - 0.5644114017486572, - 1, - 1712565012.5570464, - 1712565013.1220315, - {} - ], - [ - 9, - null, - 398764591, - null, - 0.046202414113277634, - 3.3080027103424072, - 1, - 1712565012.7521577, - 1712565016.0607708, - {} - ], - [ - 17, - null, - 398764591, - null, - 0.2303899721448468, - 0.8374693393707275, - 1, - 1712565013.0987794, - 1712565013.9371634, - {} - ], - [ - 18, - null, - 398764591, - null, - 0.04397709687403284, - 1.4642667770385742, - 1, - 1712565013.1837516, - 1712565014.6489024, - {} - ], - [ - 21, - null, - 209652396, - null, - 0.8102460538532962, - 1.399742841720581, - 1, - 1712565013.24768, - 1712565014.6482792, - {} - ], - [ - 19, - null, - 398764591, - null, - 0.04674558960074293, - 0.9420022964477539, - 1, - 1712565013.938809, - 1712565014.881975, - {} - ], - [ - 20, - null, - 398764591, - null, - 0.023378211080160916, - 1.4037482738494873, - 1, - 1712565014.003451, - 1712565015.4077935, - {} - ], - [ - 17, - null, - 924231285, - null, - 0.2531925100588053, - 0.8451352119445801, - 1, - 1712565014.7065277, - 1712565015.5526712, - {} - ], - [ - 16, - null, - 398764591, - null, - 0.02560662333642827, - 3.018314838409424, - 1, - 1712565014.7788284, - 1712565017.7980478, - {} - ], - [ - 18, - null, - 924231285, - null, - 0.033389043639740046, - 1.1432137489318848, - 1, - 1712565014.83027, - 1712565015.9741292, - {} - ], - [ - 21, - null, - 398764591, - null, - 0.8770318786753327, - 1.9530634880065918, - 1, - 1712565014.9338968, - 1712565016.8878474, - {} - ], - [ - 19, - null, - 924231285, - null, - 0.07066542865985759, - 1.0578465461730957, - 1, - 1712565015.468166, - 1712565016.5266423, - {} - ], - [ - 20, - null, - 924231285, - null, - 0.02059114825131536, - 2.661302089691162, - 1, - 1712565015.615147, - 1712565018.2773883, - {} - ], - [ - 22, - null, - 209652396, - null, - 0.04287062828845567, - 1.5211269855499268, - 1, - 1712565016.0558727, - 1712565017.5779333, - {} - ], - [ - 23, - null, - 209652396, - null, - 0.04841999380996587, - 8.354167461395264, - 1, - 1712565016.1641889, - 1712565024.5193043, - {} - ], - [ - 9, - null, - 924231285, - null, - 0.0456313834726092, - 3.8456690311431885, - 1, - 1712565016.6118283, - 1712565020.459012, - {} - ], - [ - 24, - null, - 209652396, - null, - 0.8008805323429279, - 0.46524930000305176, - 1, - 1712565016.9718251, - 1712565017.438463, - {} - ], - [ - 21, - null, - 924231285, - null, - 0.8920519962859795, - 2.354919672012329, - 1, - 1712565017.5199628, - 1712565019.8761735, - {} - ], - [ - 24, - null, - 398764591, - null, - 0.726819870009285, - 0.6692235469818115, - 1, - 1712565017.6583474, - 1712565018.328519, - {} - ], - [ - 22, - null, - 398764591, - null, - 0.032836583101207184, - 1.1873645782470703, - 1, - 1712565017.88121, - 1712565019.0695643, - {} - ], - [ - 16, - null, - 924231285, - null, - 0.02894769421231813, - 3.8686747550964355, - 1, - 1712565018.3620129, - 1712565022.2322352, - {} - ], - [ - 24, - null, - 924231285, - null, - 0.7039430516867843, - 0.5134823322296143, - 1, - 1712565018.45915, - 1712565018.9735632, - {} - ], - [ - 25, - null, - 209652396, - null, - 0.0473026926648098, - 16.41287326812744, - 1, - 1712565019.0678082, - 1712565035.4815993, - {} - ], - [ - 11, - null, - 398764591, - null, - 0.032836583101207184, - 11.42868709564209, - 1, - 1712565019.1586893, - 1712565030.5886083, - {} - ], - [ - 22, - null, - 924231285, - null, - 0.042299597647787124, - 1.5826566219329834, - 1, - 1712565019.2559285, - 1712565020.8400044, - {} - ], - [ - 26, - null, - 209652396, - null, - 0.7974450634478489, - 0.5728342533111572, - 1, - 1712565019.9510424, - 1712565020.5249114, - {} - ], - [ - 27, - null, - 209652396, - null, - 0.09015010832559567, - 4.364055395126343, - 1, - 1712565020.3812418, - 1712565024.7467766, - {} - ], - [ - 5, - null, - 398764591, - null, - 0.9003884246363355, - 14.183544158935547, - 1, - 1712565020.5460055, - 1712565034.730529, - {} - ], - [ - 28, - null, - 209652396, - null, - 0.0984989167440421, - 0.9867184162139893, - 1, - 1712565020.6601338, - 1712565021.6482403, - {} - ], - [ - 26, - null, - 398764591, - null, - 0.6844769421231817, - 0.5313403606414795, - 1, - 1712565020.927448, - 1712565021.4603171, - {} - ], - [ - 29, - null, - 209652396, - null, - 0.8981615598885794, - 0.22849583625793457, - 1, - 1712565021.561606, - 1712565021.7911196, - {} - ], - [ - 26, - null, - 924231285, - null, - 0.823571649644073, - 0.731696605682373, - 1, - 1712565021.7452338, - 1712565022.4780235, - {} - ], - [ - 28, - null, - 398764591, - null, - 0.10796193129062215, - 1.0691678524017334, - 1, - 1712565021.888155, - 1712565022.9592311, - {} - ], - [ - 29, - null, - 398764591, - null, - 0.89983286908078, - 0.24839043617248535, - 1, - 1712565022.327761, - 1712565022.57766, - {} - ], - [ - 30, - null, - 209652396, - null, - 0.8992742185082018, - 10.086802959442139, - 1, - 1712565022.5642378, - 1712565032.651993, - {} - ], - [ - 31, - null, - 209652396, - null, - 0.1001562983596409, - 1.2912817001342773, - 1, - 1712565022.6596029, - 1712565023.9524426, - {} - ], - [ - 29, - null, - 924231285, - null, - 0.9059610027855153, - 0.22541332244873047, - 1, - 1712565023.0410864, - 1712565023.268075, - {} - ], - [ - 28, - null, - 924231285, - null, - 0.09126121943670695, - 1.0557401180267334, - 1, - 1712565023.348831, - 1712565024.4059231, - {} - ], - [ - 32, - null, - 209652396, - null, - 0.14357319715258432, - 0.7567057609558105, - 1, - 1712565024.0348003, - 1712565024.7930315, - {} - ], - [ - 31, - null, - 398764591, - null, - 0.12243268337975866, - 0.9943161010742188, - 1, - 1712565024.4760678, - 1712565025.4713368, - {} - ], - [ - 33, - null, - 209652396, - null, - 0.16188022284122572, - 25.683335304260254, - 1, - 1712565024.600618, - 1712565050.285301, - {} - ], - [ - 23, - null, - 398764591, - null, - 0.051753327143299166, - 7.385228157043457, - 1, - 1712565024.8224359, - 1712565032.2086697, - {} - ], - [ - 27, - null, - 398764591, - null, - 0.1191024450634478, - 3.959066867828369, - 1, - 1712565024.9297316, - 1712565028.8901627, - {} - ], - [ - 32, - null, - 398764591, - null, - 0.135792324357784, - 0.838543176651001, - 1, - 1712565025.5465531, - 1712565026.3867607, - {} - ], - [ - 31, - null, - 924231285, - null, - 0.10852986691426803, - 1.264732837677002, - 1, - 1712565026.4774003, - 1712565027.743719, - {} - ], - [ - 32, - null, - 924231285, - null, - 0.1413370473537604, - 0.9844629764556885, - 1, - 1712565027.8312953, - 1712565028.8173516, - {} - ], - [ - 34, - null, - 209652396, - null, - 0.14081089445991957, - 0.7884180545806885, - 1, - 1712565028.910618, - 1712565029.700526, - {} - ], - [ - 27, - null, - 924231285, - null, - 0.10739863819251005, - 4.667412996292114, - 1, - 1712565029.015657, - 1712565033.6845143, - {} - ], - [ - 35, - null, - 209652396, - null, - 0.04230114515629835, - 3.2498931884765625, - 1, - 1712565029.786574, - 1712565033.0380797, - {} - ], - [ - 34, - null, - 398764591, - null, - 0.16361188486536682, - 0.6655764579772949, - 1, - 1712565030.678432, - 1712565031.34571, - {} - ], - [ - 11, - null, - 924231285, - null, - 0.03283194057567318, - 12.322223424911499, - 1, - 1712565031.4208908, - 1712565043.7444239, - {} - ], - [ - 34, - null, - 924231285, - null, - 0.10627050448777475, - 0.6005818843841553, - 1, - 1712565032.2954042, - 1712565032.8974326, - {} - ], - [ - 23, - null, - 924231285, - null, - 0.06121324667285677, - 7.619780540466309, - 1, - 1712565032.744003, - 1712565040.3652062, - {} - ], - [ - 30, - null, - 398764591, - null, - 0.8948297740637574, - 11.035215616226196, - 1, - 1712565032.9881554, - 1712565044.0248187, - {} - ], - [ - 36, - null, - 209652396, - null, - 0.33456979263385933, - 0.9639077186584473, - 1, - 1712565033.1345205, - 1712565034.1002426, - {} - ], - [ - 35, - null, - 398764591, - null, - 0.04396007428040849, - 3.2995266914367676, - 1, - 1712565033.7999034, - 1712565037.1014593, - {} - ], - [ - 37, - null, - 209652396, - null, - 0.8992773135252244, - 0.7176535129547119, - 1, - 1712565034.1942468, - 1712565034.9135132, - {} - ], - [ - 36, - null, - 398764591, - null, - 0.5536861652739089, - 0.7682046890258789, - 1, - 1712565034.8139176, - 1712565035.583699, - {} - ], - [ - 5, - null, - 924231285, - null, - 0.9003899721448468, - 14.44285535812378, - 1, - 1712565035.018067, - 1712565049.4626539, - {} - ], - [ - 37, - null, - 398764591, - null, - 0.8987202104611576, - 0.7350404262542725, - 1, - 1712565035.604154, - 1712565036.3408573, - {} - ], - [ - 25, - null, - 398764591, - null, - 0.04230114515629835, - 16.373348712921143, - 1, - 1712565035.7092667, - 1712565052.084024, - {} - ], - [ - 36, - null, - 924231285, - null, - 0.4841782729805014, - 0.6209244728088379, - 1, - 1712565036.4488175, - 1712565037.071179, - {} - ], - [ - 37, - null, - 924231285, - null, - 0.89983286908078, - 0.8147656917572021, - 1, - 1712565037.180723, - 1712565037.9965858, - {} - ], - [ - 35, - null, - 924231285, - null, - 0.03784122562674097, - 2.6988790035247803, - 1, - 1712565037.301929, - 1712565040.002422, - {} - ], - [ - 38, - null, - 209652396, - null, - 0.8992757660167131, - 2.005662679672241, - 1, - 1712565038.0986674, - 1712565040.10565, - {} - ], - [ - 39, - null, - 209652396, - null, - 0.10461931290622106, - 1.514883041381836, - 1, - 1712565040.0788782, - 1712565041.5947666, - {} - ], - [ - 40, - null, - 209652396, - null, - 0.023930671618693777, - 2.3705286979675293, - 1, - 1712565040.184661, - 1712565042.556817, - {} - ], - [ - 38, - null, - 398764591, - null, - 0.8987186629526462, - 1.8411645889282227, - 1, - 1712565040.4387646, - 1712565042.2815857, - {} - ], - [ - 41, - null, - 209652396, - null, - 0.5219870009285051, - 2.545016050338745, - 1, - 1712565041.699431, - 1712565044.2464228, - {} - ], - [ - 39, - null, - 398764591, - null, - 0.06066233364283513, - 1.1961593627929688, - 1, - 1712565042.3568745, - 1712565043.5545325, - {} - ], - [ - 38, - null, - 924231285, - null, - 0.8987186629526462, - 2.093280792236328, - 1, - 1712565042.63884, - 1712565044.7334743, - {} - ], - [ - 40, - null, - 398764591, - null, - 0.02560352831940571, - 2.676013469696045, - 1, - 1712565043.6318994, - 1712565046.3088646, - {} - ], - [ - 39, - null, - 924231285, - null, - 0.07399566697616833, - 1.149909496307373, - 1, - 1712565043.8234403, - 1712565044.9747822, - {} - ], - [ - 42, - null, - 209652396, - null, - 0.041186939028164615, - 1.3158142566680908, - 1, - 1712565044.1248553, - 1712565045.4421346, - {} - ], - [ - 30, - null, - 924231285, - null, - 0.9009470752089137, - 10.608931303024292, - 1, - 1712565044.3304265, - 1712565054.9409497, - {} - ], - [ - 41, - null, - 398764591, - null, - 0.4590931600123801, - 2.614492177963257, - 1, - 1712565044.812999, - 1712565047.429369, - {} - ], - [ - 43, - null, - 209652396, - null, - 0.12633859486227172, - 1.004054069519043, - 1, - 1712565045.064711, - 1712565046.0705123, - {} - ], - [ - 44, - null, - 209652396, - null, - 0.15694367069018877, - 6.814570188522339, - 1, - 1712565045.5238674, - 1712565052.3401663, - {} - ], - [ - 42, - null, - 398764591, - null, - 0.03673320953265247, - 1.7806165218353271, - 1, - 1712565046.185416, - 1712565047.967166, - {} - ], - [ - 43, - null, - 398764591, - null, - 0.16193283813060977, - 1.155543565750122, - 1, - 1712565046.4128385, - 1712565047.570328, - {} - ], - [ - 40, - null, - 924231285, - null, - 0.024484679665738307, - 2.6976280212402344, - 1, - 1712565047.529584, - 1712565050.2290654, - {} - ], - [ - 41, - null, - 924231285, - null, - 0.3995605075827917, - 1.997565507888794, - 1, - 1712565047.6774108, - 1712565049.6766438, - {} - ], - [ - 43, - null, - 924231285, - null, - 0.13798978644382542, - 0.9540266990661621, - 1, - 1712565048.050342, - 1712565049.0060203, - {} - ], - [ - 42, - null, - 924231285, - null, - 0.03227793252862887, - 1.4512040615081787, - 1, - 1712565049.0885086, - 1712565050.5413556, - {} - ], - [ - 45, - null, - 209652396, - null, - 0.8753203342618384, - 0.847496747970581, - 1, - 1712565049.543191, - 1712565050.392285, - {} - ], - [ - 46, - null, - 209652396, - null, - 0.2489322191272052, - 0.6393148899078369, - 1, - 1712565049.7643716, - 1712565050.4047034, - {} - ], - [ - 47, - null, - 209652396, - null, - 0.08013927576601676, - 15.927346467971802, - 1, - 1712565050.3042223, - 1712565066.232525, - {} - ], - [ - 33, - null, - 398764591, - null, - 0.1925379139585267, - 23.66932439804077, - 1, - 1712565050.3959663, - 1712565074.0666122, - {} - ], - [ - 48, - null, - 209652396, - null, - 0.8124187558031569, - 0.910470724105835, - 1, - 1712565050.4891617, - 1712565051.4010818, - {} - ], - [ - 45, - null, - 398764591, - null, - 0.8914763231197772, - 0.7565186023712158, - 1, - 1712565050.623894, - 1712565051.3820372, - {} - ], - [ - 46, - null, - 398764591, - null, - 0.35610492107706604, - 0.8270492553710938, - 1, - 1712565050.7091923, - 1712565051.5371573, - {} - ], - [ - 49, - null, - 209652396, - null, - 0.2715707211389663, - 6.957829475402832, - 1, - 1712565051.466912, - 1712565058.4263031, - {} - ], - [ - 48, - null, - 398764591, - null, - 0.764634787991334, - 0.8151421546936035, - 1, - 1712565051.5597835, - 1712565052.3764203, - {} - ], - [ - 45, - null, - 924231285, - null, - 0.8747632311977716, - 0.941584587097168, - 1, - 1712565051.6266928, - 1712565052.5692742, - {} - ], - [ - 46, - null, - 924231285, - null, - 0.10686939028164644, - 0.5962722301483154, - 1, - 1712565052.165133, - 1712565052.762847, - {} - ], - [ - 25, - null, - 924231285, - null, - 0.04730424017332102, - 16.94296383857727, - 1, - 1712565052.4514196, - 1712565069.3960361, - {} - ], - [ - 44, - null, - 398764591, - null, - 0.18422624574435154, - 6.634417295455933, - 1, - 1712565052.544118, - 1712565059.1801126, - {} - ], - [ - 48, - null, - 924231285, - null, - 0.8602785515320335, - 0.7708423137664795, - 1, - 1712565052.6536405, - 1712565053.4260578, - {} - ], - [ - 50, - null, - 209652396, - null, - 0.04842463633549987, - 0.9821035861968994, - 1, - 1712565052.8549998, - 1712565053.8382864, - {} - ], - [ - 51, - null, - 209652396, - null, - 0.03673475704116369, - 10.69734239578247, - 1, - 1712565053.5026176, - 1712565064.2009585, - {} - ], - [ - 52, - null, - 209652396, - null, - 0.9031723924481585, - 0.40129661560058594, - 1, - 1712565053.932928, - 1712565054.335735, - {} - ], - [ - 50, - null, - 398764591, - null, - 0.04452955741256592, - 0.7838237285614014, - 1, - 1712565054.4255831, - 1712565055.2110846, - {} - ], - [ - 52, - null, - 398764591, - null, - 0.9031723924481585, - 0.40575075149536133, - 1, - 1712565055.0314689, - 1712565055.4387197, - {} - ], - [ - 53, - null, - 209652396, - null, - 0.8987202104611576, - 0.95816969871521, - 1, - 1712565055.2899451, - 1712565056.2494361, - {} - ], - [ - 50, - null, - 924231285, - null, - 0.04896626431445372, - 0.717515230178833, - 1, - 1712565055.52647, - 1712565056.2451823, - {} - ], - [ - 52, - null, - 924231285, - null, - 0.9037264004952027, - 0.4451889991760254, - 1, - 1712565056.3189044, - 1712565056.7654727, - {} - ], - [ - 53, - null, - 398764591, - null, - 0.8992757660167131, - 1.097594976425171, - 1, - 1712565056.4288678, - 1712565057.5279367, - {} - ], - [ - 54, - null, - 209652396, - null, - 0.08237233054781812, - 1.8947386741638184, - 1, - 1712565056.8644137, - 1712565058.76014, - {} - ], - [ - 55, - null, - 209652396, - null, - 0.0801470133085731, - 1.0197317600250244, - 1, - 1712565057.613952, - 1712565058.6350822, - {} - ], - [ - 53, - null, - 924231285, - null, - 0.888133704735376, - 0.9786050319671631, - 1, - 1712565058.522568, - 1712565059.502105, - {} - ], - [ - 49, - null, - 398764591, - null, - 0.5036165273909007, - 7.000282049179077, - 1, - 1712565058.730699, - 1712565065.7326648, - {} - ], - [ - 55, - null, - 398764591, - null, - 0.08347415660786128, - 0.8997397422790527, - 1, - 1712565058.8603768, - 1712565059.7616763, - {} - ], - [ - 54, - null, - 398764591, - null, - 0.08682141751779626, - 2.5480194091796875, - 1, - 1712565059.2588532, - 1712565061.808606, - {} - ], - [ - 44, - null, - 924231285, - null, - 0.1307644692045805, - 7.21750807762146, - 1, - 1712565059.5947275, - 1712565066.8132136, - {} - ], - [ - 56, - null, - 209652396, - null, - 0.024493964716805983, - 2.156345844268799, - 1, - 1712565059.8427088, - 1712565062.0000536, - {} - ], - [ - 55, - null, - 924231285, - null, - 0.06342927886103378, - 0.9959497451782227, - 1, - 1712565061.8952117, - 1712565062.892992, - {} - ], - [ - 54, - null, - 924231285, - null, - 0.08180903744970591, - 2.1299283504486084, - 1, - 1712565062.0780683, - 1712565064.2096357, - {} - ], - [ - 56, - null, - 398764591, - null, - 0.027270194986072527, - 2.475947856903076, - 1, - 1712565062.9701085, - 1712565065.4473305, - {} - ], - [ - 57, - null, - 209652396, - null, - 0.8281089445991954, - 0.6765487194061279, - 1, - 1712565064.2821856, - 1712565064.9597, - {} - ], - [ - 51, - null, - 398764591, - null, - 0.04229495512225334, - 10.195435047149658, - 1, - 1712565064.3825853, - 1712565074.5789762, - {} - ], - [ - 58, - null, - 209652396, - null, - 0.8280609718353451, - 7.688541412353516, - 1, - 1712565065.0443525, - 1712565072.7342741, - {} - ], - [ - 57, - null, - 398764591, - null, - 0.8636691426802847, - 1.2278459072113037, - 1, - 1712565065.5273557, - 1712565066.756225, - {} - ], - [ - 56, - null, - 924231285, - null, - 0.02893995666976168, - 2.2545557022094727, - 1, - 1712565065.8200572, - 1712565068.075699, - {} - ], - [ - 49, - null, - 924231285, - null, - 0.48469978334880837, - 6.26426362991333, - 1, - 1712565066.338829, - 1712565072.6052105, - {} - ], - [ - 47, - null, - 398764591, - null, - 0.08179975239863813, - 9.90377926826477, - 1, - 1712565066.839424, - 1712565076.744739, - {} - ], - [ - 57, - null, - 924231285, - null, - 0.8580532342927887, - 0.9290714263916016, - 1, - 1712565066.9271002, - 1712565067.857914, - {} - ], - [ - 59, - null, - 209652396, - null, - 0.07180594243268335, - 0.8639552593231201, - 1, - 1712565067.9361448, - 1712565068.801029, - {} - ], - [ - 60, - null, - 209652396, - null, - 0.025601980810894487, - 1.736633539199829, - 1, - 1712565068.1614459, - 1712565069.8996112, - {} - ], - [ - 61, - null, - 209652396, - null, - 0.7572144846796658, - 4.349518537521362, - 1, - 1712565068.9006317, - 1712565073.2513459, - {} - ], - [ - 59, - null, - 398764591, - null, - 0.09125502940266172, - 1.0945045948028564, - 1, - 1712565069.482275, - 1712565070.5787363, - {} - ] - ], - "configs": { - "1": { - "activation": "logistic", - "n_layer": 4, - "n_neurons": 11, - "solver": "lbfgs" - }, - "2": { - "activation": "tanh", - "n_layer": 1, - "n_neurons": 146, - "solver": "adam", - "batch_size": 77, - "learning_rate_init": 0.000534923804864797 - }, - "3": { - "activation": "relu", - "n_layer": 5, - "n_neurons": 43, - "solver": "sgd", - "batch_size": 157, - "learning_rate": "invscaling", - "learning_rate_init": 0.08903539757637764 - }, - "4": { - "activation": "logistic", - "n_layer": 2, - "n_neurons": 90, - "solver": "adam", - "batch_size": 175, - "learning_rate_init": 0.004156370184967407 - }, - "5": { - "activation": "logistic", - "n_layer": 4, - "n_neurons": 200, - "solver": "sgd", - "batch_size": 204, - "learning_rate": "constant", - "learning_rate_init": 0.016071068084706618 - }, - "6": { - "activation": "relu", - "n_layer": 2, - "n_neurons": 12, - "solver": "sgd", - "batch_size": 119, - "learning_rate": "adaptive", - "learning_rate_init": 0.0022428359458905173 - }, - "7": { - "activation": "relu", - "n_layer": 4, - "n_neurons": 51, - "solver": "adam", - "batch_size": 31, - "learning_rate_init": 0.11617003217050809 - }, - "8": { - "activation": "tanh", - "n_layer": 2, - "n_neurons": 20, - "solver": "lbfgs" - }, - "9": { - "activation": "tanh", - "n_layer": 5, - "n_neurons": 61, - "solver": "lbfgs" - }, - "10": { - "activation": "tanh", - "n_layer": 3, - "n_neurons": 26, - "solver": "sgd", - "batch_size": 233, - "learning_rate": "adaptive", - "learning_rate_init": 0.001104760089338138 - }, - "11": { - "activation": "relu", - "n_layer": 3, - "n_neurons": 214, - "solver": "lbfgs" - }, - "12": { - "activation": "logistic", - "n_layer": 1, - "n_neurons": 18, - "solver": "adam", - "batch_size": 135, - "learning_rate_init": 0.00015265008040823863 - }, - "13": { - "activation": "logistic", - "n_layer": 4, - "n_neurons": 29, - "solver": "adam", - "batch_size": 106, - "learning_rate_init": 0.33335905507564456 - }, - "14": { - "activation": "relu", - "n_layer": 1, - "n_neurons": 84, - "solver": "lbfgs" - }, - "15": { - "activation": "tanh", - "n_layer": 5, - "n_neurons": 8, - "solver": "adam", - "batch_size": 279, - "learning_rate_init": 0.04285190354891313 - }, - "16": { - "activation": "tanh", - "n_layer": 2, - "n_neurons": 122, - "solver": "sgd", - "batch_size": 61, - "learning_rate": "constant", - "learning_rate_init": 0.00630547594663167 - }, - "17": { - "activation": "tanh", - "n_layer": 5, - "n_neurons": 23, - "solver": "adam", - "batch_size": 172, - "learning_rate_init": 0.00040121151561322107 - }, - "18": { - "activation": "tanh", - "n_layer": 2, - "n_neurons": 66, - "solver": "lbfgs" - }, - "19": { - "activation": "relu", - "n_layer": 4, - "n_neurons": 16, - "solver": "sgd", - "batch_size": 66, - "learning_rate": "constant", - "learning_rate_init": 0.005542589894513166 - }, - "20": { - "activation": "logistic", - "n_layer": 1, - "n_neurons": 232, - "solver": "sgd", - "batch_size": 253, - "learning_rate": "adaptive", - "learning_rate_init": 0.06675517414146671 - }, - "21": { - "activation": "logistic", - "n_layer": 3, - "n_neurons": 74, - "solver": "sgd", - "batch_size": 300, - "learning_rate": "adaptive", - "learning_rate_init": 0.0029904795631048244 - }, - "22": { - "activation": "relu", - "n_layer": 1, - "n_neurons": 32, - "solver": "adam", - "batch_size": 45, - "learning_rate_init": 0.012048758384365402 - }, - "23": { - "activation": "relu", - "n_layer": 5, - "n_neurons": 106, - "solver": "lbfgs" - }, - "24": { - "activation": "tanh", - "n_layer": 3, - "n_neurons": 9, - "solver": "adam", - "batch_size": 210, - "learning_rate_init": 0.1549368466139337 - }, - "25": { - "activation": "tanh", - "n_layer": 4, - "n_neurons": 158, - "solver": "sgd", - "batch_size": 143, - "learning_rate": "adaptive", - "learning_rate_init": 0.00147326235260944 - }, - "26": { - "activation": "relu", - "n_layer": 2, - "n_neurons": 10, - "solver": "sgd", - "batch_size": 193, - "learning_rate": "invscaling", - "learning_rate_init": 0.018836814953733322 - }, - "27": { - "activation": "relu", - "n_layer": 4, - "n_neurons": 97, - "solver": "adam", - "batch_size": 249, - "learning_rate_init": 0.00011444921049693927 - }, - "28": { - "activation": "logistic", - "n_layer": 2, - "n_neurons": 38, - "solver": "lbfgs" - }, - "29": { - "activation": "logistic", - "n_layer": 5, - "n_neurons": 13, - "solver": "lbfgs" - }, - "30": { - "activation": "relu", - "n_layer": 3, - "n_neurons": 175, - "solver": "adam", - "batch_size": 271, - "learning_rate_init": 0.4444919521413565 - }, - "31": { - "activation": "tanh", - "n_layer": 3, - "n_neurons": 21, - "solver": "lbfgs" - }, - "32": { - "activation": "logistic", - "n_layer": 1, - "n_neurons": 45, - "solver": "sgd", - "batch_size": 106, - "learning_rate": "invscaling", - "learning_rate_init": 0.03214161586827769 - }, - "33": { - "activation": "logistic", - "n_layer": 5, - "n_neurons": 224, - "solver": "adam", - "batch_size": 290, - "learning_rate_init": 0.004563292654888671 - }, - "34": { - "activation": "tanh", - "n_layer": 2, - "n_neurons": 17, - "solver": "lbfgs" - }, - "35": { - "activation": "relu", - "n_layer": 4, - "n_neurons": 57, - "solver": "adam", - "batch_size": 116, - "learning_rate_init": 0.00035489292891332955 - }, - "36": { - "activation": "logistic", - "n_layer": 1, - "n_neurons": 27, - "solver": "sgd", - "batch_size": 203, - "learning_rate": "adaptive", - "learning_rate_init": 0.7609797301059412 - }, - "37": { - "activation": "logistic", - "n_layer": 4, - "n_neurons": 8, - "solver": "sgd", - "batch_size": 182, - "learning_rate": "adaptive", - "learning_rate_init": 0.00019717209149247594 - }, - "38": { - "activation": "relu", - "n_layer": 2, - "n_neurons": 130, - "solver": "sgd", - "batch_size": 163, - "learning_rate": "constant", - "learning_rate_init": 0.14078837052759072 - }, - "39": { - "activation": "relu", - "n_layer": 4, - "n_neurons": 31, - "solver": "lbfgs" - }, - "40": { - "activation": "tanh", - "n_layer": 2, - "n_neurons": 80, - "solver": "adam", - "batch_size": 260, - "learning_rate_init": 0.01019194380084181 - }, - "41": { - "activation": "tanh", - "n_layer": 3, - "n_neurons": 40, - "solver": "sgd", - "batch_size": 58, - "learning_rate": "adaptive", - "learning_rate_init": 0.00012594180136887576 - }, - "42": { - "activation": "tanh", - "n_layer": 1, - "n_neurons": 94, - "solver": "lbfgs" - }, - "43": { - "activation": "relu", - "n_layer": 5, - "n_neurons": 12, - "solver": "adam", - "batch_size": 232, - "learning_rate_init": 0.0017417895884476993 - }, - "44": { - "activation": "logistic", - "n_layer": 3, - "n_neurons": 137, - "solver": "lbfgs" - }, - "45": { - "activation": "logistic", - "n_layer": 5, - "n_neurons": 54, - "solver": "lbfgs" - }, - "46": { - "activation": "relu", - "n_layer": 2, - "n_neurons": 19, - "solver": "adam", - "batch_size": 185, - "learning_rate_init": 0.039036592000729606 - }, - "47": { - "activation": "tanh", - "n_layer": 4, - "n_neurons": 191, - "solver": "sgd", - "batch_size": 240, - "learning_rate": "constant", - "learning_rate_init": 0.0007368727861046234 - }, - "48": { - "activation": "tanh", - "n_layer": 1, - "n_neurons": 13, - "solver": "sgd", - "batch_size": 88, - "learning_rate": "adaptive", - "learning_rate_init": 0.5025399902870405 - }, - "49": { - "activation": "logistic", - "n_layer": 4, - "n_neurons": 102, - "solver": "lbfgs" - }, - "50": { - "activation": "tanh", - "n_layer": 1, - "n_neurons": 35, - "solver": "adam", - "batch_size": 129, - "learning_rate_init": 0.0034213963211558564 - }, - "51": { - "activation": "relu", - "n_layer": 5, - "n_neurons": 148, - "solver": "lbfgs" - }, - "52": { - "activation": "logistic", - "n_layer": 2, - "n_neurons": 10, - "solver": "sgd", - "batch_size": 294, - "learning_rate": "constant", - "learning_rate_init": 0.00047328481860027055 - }, - "53": { - "activation": "logistic", - "n_layer": 5, - "n_neurons": 20, - "solver": "adam", - "batch_size": 256, - "learning_rate_init": 0.10560248678966794 - }, - "54": { - "activation": "relu", - "n_layer": 3, - "n_neurons": 48, - "solver": "sgd", - "batch_size": 68, - "learning_rate": "adaptive", - "learning_rate_init": 0.00026295202043791785 - }, - "55": { - "activation": "tanh", - "n_layer": 3, - "n_neurons": 14, - "solver": "adam", - "batch_size": 150, - "learning_rate_init": 0.013590087590816565 - }, - "56": { - "activation": "tanh", - "n_layer": 1, - "n_neurons": 167, - "solver": "lbfgs" - }, - "57": { - "activation": "tanh", - "n_layer": 5, - "n_neurons": 15, - "solver": "sgd", - "batch_size": 99, - "learning_rate": "constant", - "learning_rate_init": 0.2144498574918249 - }, - "58": { - "activation": "relu", - "n_layer": 2, - "n_neurons": 243, - "solver": "sgd", - "batch_size": 217, - "learning_rate": "invscaling", - "learning_rate_init": 0.00016793749976650653 - }, - "59": { - "activation": "relu", - "n_layer": 4, - "n_neurons": 24, - "solver": "lbfgs" - }, - "60": { - "activation": "logistic", - "n_layer": 1, - "n_neurons": 62, - "solver": "adam", - "batch_size": 51, - "learning_rate_init": 0.0013063520380558128 - }, - "61": { - "activation": "logistic", - "n_layer": 3, - "n_neurons": 114, - "solver": "adam", - "batch_size": 90, - "learning_rate_init": 0.0520608529688821 - } - }, - "config_origins": { - "1": "Initial Design: Sobol", - "2": "Initial Design: Sobol", - "3": "Initial Design: Sobol", - "4": "Initial Design: Sobol", - "5": "Initial Design: Sobol", - "6": "Initial Design: Sobol", - "7": "Initial Design: Sobol", - "8": "Initial Design: Sobol", - "9": "Initial Design: Sobol", - "10": "Initial Design: Sobol", - "11": "Initial Design: Sobol", - "12": "Initial Design: Sobol", - "13": "Initial Design: Sobol", - "14": "Initial Design: Sobol", - "15": "Initial Design: Sobol", - "16": "Initial Design: Sobol", - "17": "Initial Design: Sobol", - "18": "Initial Design: Sobol", - "19": "Initial Design: Sobol", - "20": "Initial Design: Sobol", - "21": "Initial Design: Sobol", - "22": "Initial Design: Sobol", - "23": "Initial Design: Sobol", - "24": "Initial Design: Sobol", - "25": "Initial Design: Sobol", - "26": "Initial Design: Sobol", - "27": "Initial Design: Sobol", - "28": "Initial Design: Sobol", - "29": "Initial Design: Sobol", - "30": "Initial Design: Sobol", - "31": "Initial Design: Sobol", - "32": "Initial Design: Sobol", - "33": "Initial Design: Sobol", - "34": "Initial Design: Sobol", - "35": "Initial Design: Sobol", - "36": "Initial Design: Sobol", - "37": "Initial Design: Sobol", - "38": "Initial Design: Sobol", - "39": "Initial Design: Sobol", - "40": "Initial Design: Sobol", - "41": "Initial Design: Sobol", - "42": "Initial Design: Sobol", - "43": "Initial Design: Sobol", - "44": "Initial Design: Sobol", - "45": "Initial Design: Sobol", - "46": "Initial Design: Sobol", - "47": "Initial Design: Sobol", - "48": "Initial Design: Sobol", - "49": "Initial Design: Sobol", - "50": "Initial Design: Sobol", - "51": "Initial Design: Sobol", - "52": "Initial Design: Sobol", - "53": "Initial Design: Sobol", - "54": "Initial Design: Sobol", - "55": "Initial Design: Sobol", - "56": "Initial Design: Sobol", - "57": "Initial Design: Sobol", - "58": "Initial Design: Sobol", - "59": "Initial Design: Sobol", - "60": "Initial Design: Sobol", - "61": "Initial Design: Sobol" - } -} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_1/scenario 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_1/scenario 2.json deleted file mode 100644 index d6ceef93..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_1/scenario 2.json +++ /dev/null @@ -1,155 +0,0 @@ -{ - "name": "bcdb4a7cedc37b0caa608ab570d77c03", - "deterministic": false, - "objectives": "cost", - "crash_cost": Infinity, - "termination_cost_threshold": Infinity, - "walltime_limit": 60, - "cputime_limit": Infinity, - "trial_walltime_limit": null, - "trial_memory_limit": null, - "n_trials": 500, - "use_default_config": false, - "instances": null, - "instance_features": null, - "min_budget": null, - "max_budget": null, - "seed": 0, - "n_workers": 8, - "_meta": { - "facade": { - "name": "HyperparameterOptimizationFacade" - }, - "runner": { - "name": "DaskParallelRunner" - }, - "model": { - "name": "RandomForest", - "types": [ - 3, - 0, - 0, - 3, - 0, - 4, - 0 - ], - "bounds": [ - [ - 3, - NaN - ], - [ - 0, - 1.0 - ], - [ - 0, - 1.0 - ], - [ - 3, - NaN - ], - [ - -1.0, - 1.0 - ], - [ - 4, - NaN - ], - [ - -1.0, - 1.0 - ] - ], - "pca_components": 7, - "n_trees": 10, - "n_points_per_tree": -1, - "ratio_features": 1.0, - "min_samples_split": 2, - "min_samples_leaf": 1, - "max_depth": 1048576, - "eps_purity": 1e-08, - "max_nodes": 1048576, - "bootstrapping": true - }, - "acquisition_maximizer": { - "name": "LocalAndSortedRandomSearch", - "acquisition_function": { - "name": "EI", - "xi": 0.0, - "log": true - }, - "challengers": 10000, - "seed": 0, - "random_search": { - "name": "RandomSearch", - "acquisition_function": { - "name": "EI", - "xi": 0.0, - "log": true - }, - "challengers": 5000, - "seed": 0 - }, - "local_search": { - "name": "LocalSearch", - "acquisition_function": { - "name": "EI", - "xi": 0.0, - "log": true - }, - "challengers": 5000, - "seed": 0, - "max_steps": null, - "n_steps_plateau_walk": 10, - "vectorization_min_obtain": 2, - "vectorization_max_obtain": 64 - } - }, - "acquisition_function": { - "name": "EI", - "xi": 0.0, - "log": true - }, - "intensifier": { - "name": "Intensifier", - "max_incumbents": 10, - "seed": 0 - }, - "initial_design": { - "name": "SobolInitialDesign", - "n_configs": 70, - "n_configs_per_hyperparameter": 10, - "additional_configs": [], - "seed": 0 - }, - "random_design": { - "name": "ProbabilityRandomDesign", - "seed": 0, - "probability": 0.2 - }, - "runhistory_encoder": { - "name": "RunHistoryLogScaledEncoder", - "considered_states": [ - 1, - 2, - 4 - ], - "lower_budget_states": [], - "scale_percentage": 5, - "seed": 0 - }, - "multi_objective_algorithm": null, - "config_selector": { - "name": "ConfigSelector", - "retrain_after": 8, - "retries": 16, - "min_trials": 1 - }, - "version": "2.0.2" - }, - "output_directory": "smac3_output/bcdb4a7cedc37b0caa608ab570d77c03/0" -} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/configspace 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/configspace 2.json deleted file mode 100644 index be8aa231..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_2/configspace 2.json +++ /dev/null @@ -1 +0,0 @@ -{"name": null, "hyperparameters": [{"type": "categorical", "name": "activation", "choices": ["logistic", "tanh", "relu"], "weights": null, "default_value": "tanh", "meta": null}, {"type": "uniform_int", "name": "n_layer", "lower": 1, "upper": 5, "default_value": 1, "log": false, "meta": null}, {"type": "uniform_int", "name": "n_neurons", "lower": 8, "upper": 256, "default_value": 10, "log": true, "meta": null}, {"type": "categorical", "name": "solver", "choices": ["lbfgs", "sgd", "adam"], "weights": null, "default_value": "adam", "meta": null}, {"type": "uniform_int", "name": "batch_size", "lower": 30, "upper": 300, "default_value": 200, "log": false, "meta": null}, {"type": "categorical", "name": "learning_rate", "choices": ["constant", "invscaling", "adaptive"], "weights": null, "default_value": "constant", "meta": null}, {"type": "uniform_float", "name": "learning_rate_init", "lower": 0.0001, "upper": 1.0, "default_value": 0.001, "log": true, "meta": null}], "conditions": [{"type": "IN", "child": "batch_size", "parent": "solver", "values": ["sgd", "adam"]}, {"type": "EQ", "child": "learning_rate", "parent": "solver", "value": "sgd"}, {"type": "IN", "child": "learning_rate_init", "parent": "solver", "values": ["sgd", "adam"]}], "forbiddens": [], "python_module_version": "1.2.0", "format_version": 0.4} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/intensifier 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/intensifier 2.json deleted file mode 100644 index 57b6338e..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_2/intensifier 2.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "incumbent_ids": [ - 4 - ], - "rejected_config_ids": [ - 8, - 1, - 7, - 3, - 13, - 10, - 14, - 12, - 2, - 17, - 6, - 15, - 16, - 9 - ], - "incumbents_changed": 3, - "trajectory": [ - { - "config_ids": [ - 1 - ], - "costs": [ - 0.6699473847106159 - ], - "trial": 1, - "walltime": 0.7647695541381836 - }, - { - "config_ids": [ - 10 - ], - "costs": [ - 0.05601722892809243 - ], - "trial": 14, - "walltime": 5.185967445373535 - }, - { - "config_ids": [ - 4 - ], - "costs": [ - 0.02671515526668729 - ], - "trial": 28, - "walltime": 9.271684408187866 - } - ], - "state": { - "queue": [ - [ - 5, - 2 - ], - [ - 19, - 2 - ], - [ - 11, - 4 - ], - [ - 18, - 4 - ] - ] - } -} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/optimization 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/optimization 2.json deleted file mode 100644 index 445f0dfa..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_2/optimization 2.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "used_walltime": 15.669282674789429, - "used_target_function_walltime": 102.76791548728943, - "last_update": 1706277639.8103335, - "finished": true -} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/runhistory 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/runhistory 2.json deleted file mode 100644 index 5be2d9d8..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_2/runhistory 2.json +++ /dev/null @@ -1,766 +0,0 @@ -{ - "stats": { - "submitted": 50, - "finished": 50, - "running": 0 - }, - "data": [ - [ - 1, - null, - 209652396, - null, - 0.6699473847106159, - 0.6026918888092041, - 1, - 1706277624.222565, - 1706277624.8264945, - {} - ], - [ - 2, - null, - 209652396, - null, - 0.025597338285360594, - 3.1097147464752197, - 1, - 1706277624.2967758, - 1706277627.407669, - {} - ], - [ - 3, - null, - 209652396, - null, - 0.6583333333333333, - 2.336819648742676, - 1, - 1706277624.3608909, - 1706277626.6990438, - {} - ], - [ - 4, - null, - 209652396, - null, - 0.02560352831940571, - 2.9831438064575195, - 1, - 1706277624.415114, - 1706277627.3992622, - {} - ], - [ - 5, - null, - 209652396, - null, - 0.8992773135252243, - 14.455097913742065, - 1, - 1706277624.4994059, - 1706277638.9561086, - {} - ], - [ - 6, - null, - 209652396, - null, - 0.3678195605075827, - 4.080667018890381, - 1, - 1706277624.5847695, - 1706277628.6664875, - {} - ], - [ - 7, - null, - 209652396, - null, - 0.8992757660167131, - 1.912712574005127, - 1, - 1706277624.6662445, - 1706277626.5800583, - {} - ], - [ - 8, - null, - 209652396, - null, - 0.8241210151655834, - 0.5521142482757568, - 1, - 1706277624.7443416, - 1706277625.2985408, - {} - ], - [ - 9, - null, - 209652396, - null, - 0.0317254719900959, - 5.659100532531738, - 1, - 1706277624.910818, - 1706277630.5716987, - {} - ], - [ - 1, - null, - 398764591, - null, - 0.6861915815536985, - 0.8483016490936279, - 1, - 1706277625.409777, - 1706277626.2611463, - {} - ], - [ - 10, - null, - 209652396, - null, - 0.056199319096254974, - 0.7128767967224121, - 1, - 1706277626.3443322, - 1706277627.058281, - {} - ], - [ - 1, - null, - 924231285, - null, - 0.7017641597028783, - 0.723477840423584, - 1, - 1706277626.6957843, - 1706277627.4203732, - {} - ], - [ - 7, - null, - 398764591, - null, - 0.8987202104611576, - 1.744448184967041, - 1, - 1706277626.7947023, - 1706277628.5401833, - {} - ], - [ - 3, - null, - 398764591, - null, - 0.8458000619003405, - 1.7513632774353027, - 1, - 1706277627.1429644, - 1706277628.896084, - {} - ], - [ - 10, - null, - 398764591, - null, - 0.056761064685855844, - 0.6708414554595947, - 1, - 1706277627.5016356, - 1706277628.1742969, - {} - ], - [ - 2, - null, - 398764591, - null, - 0.03006035283194064, - 3.3852593898773193, - 1, - 1706277627.64613, - 1706277631.0330572, - {} - ], - [ - 4, - null, - 398764591, - null, - 0.027273290003095085, - 2.520305633544922, - 1, - 1706277627.740861, - 1706277630.2628653, - {} - ], - [ - 11, - null, - 209652396, - null, - 0.03840916125038696, - 7.205214738845825, - 1, - 1706277628.2559705, - 1706277635.4626877, - {} - ], - [ - 10, - null, - 924231285, - null, - 0.05509130300216647, - 0.6167201995849609, - 1, - 1706277628.640889, - 1706277629.2592533, - {} - ], - [ - 7, - null, - 924231285, - null, - 0.9009470752089137, - 1.3385493755340576, - 1, - 1706277628.76265, - 1706277630.102818, - {} - ], - [ - 6, - null, - 398764591, - null, - 0.398457134014237, - 3.8393754959106445, - 1, - 1706277628.9893534, - 1706277632.830835, - {} - ], - [ - 3, - null, - 924231285, - null, - Infinity, - 0.0, - 2, - 1706277629.3351586, - 1706277630.2660835, - { - "traceback": "Traceback (most recent call last):\n File \"/home/skrebs/Projects/Software/SMAC3/smac/runner/target_function_runner.py\", line 184, in run\n rval = self(config_copy, target_function, kwargs)\n File \"/home/skrebs/Projects/Software/SMAC3/smac/runner/target_function_runner.py\", line 257, in __call__\n return algorithm(config, **algorithm_kwargs)\n File \"/home/skrebs/Projects/Software/SMAC3/examples/2_multi_fidelity/1_mlp_epochs.py\", line 107, in train\n score = cross_val_score(classifier, dataset.data, dataset.target, cv=cv, error_score=\"raise\")\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/model_selection/_validation.py\", line 562, in cross_val_score\n cv_results = cross_validate(\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/utils/_param_validation.py\", line 211, in wrapper\n return func(*args, **kwargs)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/model_selection/_validation.py\", line 309, in cross_validate\n results = parallel(\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/utils/parallel.py\", line 65, in __call__\n return super().__call__(iterable_with_config)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 1088, in __call__\n while self.dispatch_one_batch(iterator):\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 901, in dispatch_one_batch\n self._dispatch(tasks)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 819, in _dispatch\n job = self._backend.apply_async(batch, callback=cb)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/_parallel_backends.py\", line 208, in apply_async\n result = ImmediateResult(func)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/_parallel_backends.py\", line 597, in __init__\n self.results = batch()\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 288, in __call__\n return [func(*args, **kwargs)\n File \"/home/skrebs/.local/lib/python3.10/site-packages/joblib/parallel.py\", line 288, in \n return [func(*args, **kwargs)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/utils/parallel.py\", line 127, in __call__\n return self.function(*args, **kwargs)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/model_selection/_validation.py\", line 732, in _fit_and_score\n estimator.fit(X_train, y_train, **fit_params)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/base.py\", line 1151, in wrapper\n return fit_method(estimator, *args, **kwargs)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py\", line 753, in fit\n return self._fit(X, y, incremental=False)\n File \"/home/skrebs/miniconda3/envs/SMAC/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py\", line 496, in _fit\n raise ValueError(\nValueError: Solver produced non-finite parameter weights. The input data may contain large values and need to be preprocessed.\n", - "error": "ValueError('Solver produced non-finite parameter weights. The input data may contain large values and need to be preprocessed.')" - } - ], - [ - 12, - null, - 209652396, - null, - 0.10016403590219747, - 1.307952642440796, - 1, - 1706277630.1908402, - 1706277631.4997926, - {} - ], - [ - 13, - null, - 209652396, - null, - 0.89983286908078, - 0.2807650566101074, - 1, - 1706277630.3577998, - 1706277630.6401331, - {} - ], - [ - 4, - null, - 924231285, - null, - 0.02726864747756108, - 2.861912250518799, - 1, - 1706277630.4543989, - 1706277633.3173153, - {} - ], - [ - 14, - null, - 209652396, - null, - 0.3895543175487466, - 0.808574914932251, - 1, - 1706277630.6644979, - 1706277631.4741547, - {} - ], - [ - 9, - null, - 398764591, - null, - 0.032271742494583755, - 5.011699914932251, - 1, - 1706277630.76806, - 1706277635.7808084, - {} - ], - [ - 13, - null, - 398764591, - null, - 0.8886908077994429, - 0.32169437408447266, - 1, - 1706277631.1505885, - 1706277631.4740863, - {} - ], - [ - 2, - null, - 924231285, - null, - 0.029497059733828435, - 2.967978000640869, - 1, - 1706277631.5600839, - 1706277634.529425, - {} - ], - [ - 12, - null, - 398764591, - null, - 0.08014391829155054, - 1.0976686477661133, - 1, - 1706277631.6530569, - 1706277632.7523105, - {} - ], - [ - 14, - null, - 398764591, - null, - 0.495255338904364, - 0.6123650074005127, - 1, - 1706277631.7122657, - 1706277632.3262582, - {} - ], - [ - 13, - null, - 924231285, - null, - 0.9048467966573817, - 0.22585248947143555, - 1, - 1706277632.4026914, - 1706277632.6302176, - {} - ], - [ - 14, - null, - 924231285, - null, - 0.31948467966573824, - 0.7365078926086426, - 1, - 1706277632.7140539, - 1706277633.4522555, - {} - ], - [ - 15, - null, - 209652396, - null, - 0.04341844630145464, - 1.4842357635498047, - 1, - 1706277632.830031, - 1706277634.3161619, - {} - ], - [ - 12, - null, - 924231285, - null, - 0.08125657691117305, - 1.5107638835906982, - 1, - 1706277632.9116373, - 1706277634.4242318, - {} - ], - [ - 6, - null, - 924231285, - null, - 0.36005261528938415, - 3.516299247741699, - 1, - 1706277633.4184535, - 1706277636.9357994, - {} - ], - [ - 16, - null, - 209652396, - null, - 0.05231352522438881, - 1.0166974067687988, - 1, - 1706277633.539396, - 1706277634.5570924, - {} - ], - [ - 17, - null, - 209652396, - null, - 0.15859950479727636, - 0.510662317276001, - 1, - 1706277634.4225478, - 1706277634.9349742, - {} - ], - [ - 15, - null, - 398764591, - null, - 0.043413803775920856, - 1.1191012859344482, - 1, - 1706277634.5313032, - 1706277635.651467, - {} - ], - [ - 18, - null, - 209652396, - null, - 0.03673320953265247, - 1.036933183670044, - 1, - 1706277635.221057, - 1706277636.2595854, - {} - ], - [ - 16, - null, - 398764591, - null, - 0.042296502630764565, - 1.2329812049865723, - 1, - 1706277635.3001854, - 1706277636.5348752, - {} - ], - [ - 17, - null, - 398764591, - null, - 0.1229944289693593, - 0.5270049571990967, - 1, - 1706277635.3790934, - 1706277635.907605, - {} - ], - [ - 19, - null, - 209652396, - null, - 0.04006809037449699, - 1.2701239585876465, - 1, - 1706277635.5504642, - 1706277636.8224409, - {} - ], - [ - 11, - null, - 398764591, - null, - 0.0395094398019189, - 3.738813638687134, - 1, - 1706277635.738447, - 1706277639.4788716, - {} - ], - [ - 15, - null, - 924231285, - null, - 0.036725471990095904, - 1.2338676452636719, - 1, - 1706277635.8669333, - 1706277637.1028805, - {} - ], - [ - 9, - null, - 924231285, - null, - 0.03617301145156304, - 3.797776222229004, - 1, - 1706277635.9984384, - 1706277639.797911, - {} - ], - [ - 17, - null, - 924231285, - null, - 0.11129990714948934, - 0.5645942687988281, - 1, - 1706277636.3598018, - 1706277636.9253795, - {} - ], - [ - 18, - null, - 398764591, - null, - 0.040072732900030994, - 1.1205317974090576, - 1, - 1706277636.6408405, - 1706277637.7630773, - {} - ], - [ - 16, - null, - 924231285, - null, - 0.03450324976787367, - 1.0402038097381592, - 1, - 1706277636.9236588, - 1706277637.9659233, - {} - ], - [ - 19, - null, - 398764591, - null, - 0.038952336737852145, - 0.7655572891235352, - 1, - 1706277637.010183, - 1706277637.7774007, - {} - ] - ], - "configs": { - "1": { - "activation": "logistic", - "n_layer": 4, - "n_neurons": 11, - "solver": "lbfgs" - }, - "2": { - "activation": "tanh", - "n_layer": 1, - "n_neurons": 146, - "solver": "adam", - "batch_size": 77, - "learning_rate_init": 0.000534923804864797 - }, - "3": { - "activation": "relu", - "n_layer": 5, - "n_neurons": 43, - "solver": "sgd", - "batch_size": 157, - "learning_rate": "invscaling", - "learning_rate_init": 0.08903539757637764 - }, - "4": { - "activation": "logistic", - "n_layer": 2, - "n_neurons": 90, - "solver": "adam", - "batch_size": 175, - "learning_rate_init": 0.004156370184967407 - }, - "5": { - "activation": "logistic", - "n_layer": 4, - "n_neurons": 200, - "solver": "sgd", - "batch_size": 204, - "learning_rate": "constant", - "learning_rate_init": 0.016071068084706618 - }, - "6": { - "activation": "tanh", - "n_layer": 4, - "n_neurons": 63, - "solver": "sgd", - "batch_size": 144, - "learning_rate": "invscaling", - "learning_rate_init": 0.0056279320474151686 - }, - "7": { - "activation": "relu", - "n_layer": 5, - "n_neurons": 29, - "solver": "adam", - "batch_size": 173, - "learning_rate_init": 0.503948959867121 - }, - "8": { - "activation": "logistic", - "n_layer": 1, - "n_neurons": 8, - "solver": "adam", - "batch_size": 240, - "learning_rate_init": 0.8212461922256864 - }, - "9": { - "activation": "relu", - "n_layer": 3, - "n_neurons": 118, - "solver": "lbfgs" - }, - "10": { - "activation": "tanh", - "n_layer": 1, - "n_neurons": 35, - "solver": "lbfgs" - }, - "11": { - "activation": "tanh", - "n_layer": 4, - "n_neurons": 80, - "solver": "lbfgs" - }, - "12": { - "activation": "tanh", - "n_layer": 3, - "n_neurons": 27, - "solver": "lbfgs" - }, - "13": { - "activation": "logistic", - "n_layer": 5, - "n_neurons": 16, - "solver": "lbfgs" - }, - "14": { - "activation": "relu", - "n_layer": 4, - "n_neurons": 12, - "solver": "lbfgs" - }, - "15": { - "activation": "tanh", - "n_layer": 1, - "n_neurons": 63, - "solver": "lbfgs" - }, - "16": { - "activation": "relu", - "n_layer": 2, - "n_neurons": 43, - "solver": "lbfgs" - }, - "17": { - "activation": "relu", - "n_layer": 2, - "n_neurons": 18, - "solver": "lbfgs" - }, - "18": { - "activation": "tanh", - "n_layer": 1, - "n_neurons": 82, - "solver": "lbfgs" - }, - "19": { - "activation": "relu", - "n_layer": 1, - "n_neurons": 57, - "solver": "lbfgs" - } - }, - "config_origins": { - "1": "Initial Design: Sobol", - "2": "Initial Design: Sobol", - "3": "Initial Design: Sobol", - "4": "Initial Design: Sobol", - "5": "Initial Design: Sobol", - "6": null, - "7": null, - "8": null, - "9": null, - "10": "Acquisition Function Maximizer: Random Search (sorted)", - "11": "Acquisition Function Maximizer: Random Search (sorted)", - "12": "Acquisition Function Maximizer: Random Search (sorted)", - "13": "Acquisition Function Maximizer: Random Search (sorted)", - "14": "Acquisition Function Maximizer: Random Search (sorted)", - "15": "Acquisition Function Maximizer: Local Search", - "16": "Acquisition Function Maximizer: Local Search", - "17": "Acquisition Function Maximizer: Local Search", - "18": "Acquisition Function Maximizer: Random Search (sorted)", - "19": "Acquisition Function Maximizer: Random Search (sorted)" - } -} \ No newline at end of file diff --git a/logs/SMAC3v2/mlp-nondeterministic/run_2/scenario 2.json b/logs/SMAC3v2/mlp-nondeterministic/run_2/scenario 2.json deleted file mode 100644 index 679a91c6..00000000 --- a/logs/SMAC3v2/mlp-nondeterministic/run_2/scenario 2.json +++ /dev/null @@ -1,155 +0,0 @@ -{ - "name": "e262f9639f7cb897f1e597cb5b717d0c", - "deterministic": false, - "objectives": "cost", - "crash_cost": Infinity, - "termination_cost_threshold": Infinity, - "walltime_limit": Infinity, - "cputime_limit": Infinity, - "trial_walltime_limit": null, - "trial_memory_limit": null, - "n_trials": 50, - "use_default_config": false, - "instances": null, - "instance_features": null, - "min_budget": null, - "max_budget": null, - "seed": 0, - "n_workers": 8, - "_meta": { - "facade": { - "name": "HyperparameterOptimizationFacade" - }, - "runner": { - "name": "DaskParallelRunner" - }, - "model": { - "name": "RandomForest", - "types": [ - 3, - 0, - 0, - 3, - 0, - 4, - 0 - ], - "bounds": [ - [ - 3, - NaN - ], - [ - 0, - 1.0 - ], - [ - 0, - 1.0 - ], - [ - 3, - NaN - ], - [ - -1.0, - 1.0 - ], - [ - 4, - NaN - ], - [ - -1.0, - 1.0 - ] - ], - "pca_components": 7, - "n_trees": 10, - "n_points_per_tree": -1, - "ratio_features": 1.0, - "min_samples_split": 2, - "min_samples_leaf": 1, - "max_depth": 1048576, - "eps_purity": 1e-08, - "max_nodes": 1048576, - "bootstrapping": true - }, - "acquisition_maximizer": { - "name": "LocalAndSortedRandomSearch", - "acquisition_function": { - "name": "EI", - "xi": 0.0, - "log": true - }, - "challengers": 10000, - "seed": 0, - "random_search": { - "name": "RandomSearch", - "acquisition_function": { - "name": "EI", - "xi": 0.0, - "log": true - }, - "challengers": 5000, - "seed": 0 - }, - "local_search": { - "name": "LocalSearch", - "acquisition_function": { - "name": "EI", - "xi": 0.0, - "log": true - }, - "challengers": 5000, - "seed": 0, - "max_steps": null, - "n_steps_plateau_walk": 10, - "vectorization_min_obtain": 2, - "vectorization_max_obtain": 64 - } - }, - "acquisition_function": { - "name": "EI", - "xi": 0.0, - "log": true - }, - "intensifier": { - "name": "Intensifier", - "max_incumbents": 10, - "seed": 0 - }, - "initial_design": { - "name": "SobolInitialDesign", - "n_configs": 5, - "n_configs_per_hyperparameter": 10, - "additional_configs": [], - "seed": 0 - }, - "random_design": { - "name": "ProbabilityRandomDesign", - "seed": 0, - "probability": 0.2 - }, - "runhistory_encoder": { - "name": "RunHistoryLogScaledEncoder", - "considered_states": [ - 1, - 2, - 4 - ], - "lower_budget_states": [], - "scale_percentage": 5, - "seed": 0 - }, - "multi_objective_algorithm": null, - "config_selector": { - "name": "ConfigSelector", - "retrain_after": 8, - "retries": 16, - "min_trials": 1 - }, - "version": "2.0.2" - }, - "output_directory": "smac3_output/e262f9639f7cb897f1e597cb5b717d0c/0" -} \ No newline at end of file diff --git a/tests/test_runs/__init__ 2.py b/tests/test_runs/__init__ 2.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/test_utils/__init__ 2.py b/tests/test_utils/__init__ 2.py deleted file mode 100644 index e69de29b..00000000 From a400d2fd1a9add55e638cb38ba04f7baee3eb6af Mon Sep 17 00:00:00 2001 From: Kristina Thieme Date: Tue, 8 Oct 2024 13:40:07 +0200 Subject: [PATCH 5/6] deleted another unnecessary file --- docs/sg_execution_times.rst | 58 ------------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 docs/sg_execution_times.rst diff --git a/docs/sg_execution_times.rst b/docs/sg_execution_times.rst deleted file mode 100644 index 1654b0ec..00000000 --- a/docs/sg_execution_times.rst +++ /dev/null @@ -1,58 +0,0 @@ - -:orphan: - -.. _sphx_glr_sg_execution_times: - - -Computation times -================= -**00:00.000** total execution time for 8 files **from all galleries**: - -.. container:: - - .. raw:: html - - - - - - - - .. list-table:: - :header-rows: 1 - :class: table table-striped sg-datatable - - * - Example - - Time - - Mem (MB) - * - :ref:`sphx_glr_examples_api_ablation_paths.py` (``../examples/api/ablation_paths.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_examples_api_importances 2.py` (``../examples/api/importances 2.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_examples_api_importances.py` (``../examples/api/importances.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_examples_api_parallel_coordinates.py` (``../examples/api/parallel_coordinates.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_examples_api_pdp.py` (``../examples/api/pdp.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_examples_record_digits_sklearn.py` (``../examples/record/digits_sklearn.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_examples_record_minimal.py` (``../examples/record/minimal.py``) - - 00:00.000 - - 0.0 - * - :ref:`sphx_glr_examples_record_mnist_pytorch.py` (``../examples/record/mnist_pytorch.py``) - - 00:00.000 - - 0.0 From 4147a5a0bace98eacacd13d3472bcacf1f4d2822 Mon Sep 17 00:00:00 2001 From: Kristina Thieme Date: Mon, 21 Oct 2024 11:43:14 +0200 Subject: [PATCH 6/6] changed spelling errors --- docs/converters/bohb.rst | 2 +- docs/converters/custom_converters.rst | 2 +- docs/converters/smac.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/converters/bohb.rst b/docs/converters/bohb.rst index 89d1ae0b..681da493 100644 --- a/docs/converters/bohb.rst +++ b/docs/converters/bohb.rst @@ -7,7 +7,7 @@ To load an BOHB run into DeepCAVE, it is necessary to install HpBandSter, e.g. v pip install deepcave[bohb] -To be able to load it correctly into DeepCAVE, you need save the configuration space as a ConfigSpace JSON file: +To be able to load it correctly into DeepCAVE, you need to save the configuration space as a ConfigSpace JSON file: .. code-block:: python diff --git a/docs/converters/custom_converters.rst b/docs/converters/custom_converters.rst index 9f5467ff..d8820a6a 100644 --- a/docs/converters/custom_converters.rst +++ b/docs/converters/custom_converters.rst @@ -71,7 +71,7 @@ Step 3: Implement the methods Inside this method, you have to provide a configspace, objectives, and meta data, as well as the trials, which have to be added to the newly created run object. -4. **is_valid_run**: This method sould return True, if the given path belongs to a valid run. Valid means, that the given run +4. **is_valid_run**: This method should return True if the given path belongs to a valid run. Valid means that the given run directory contains all files that are necessary for further processing (e.g. a configspace.json). Step 5: Document your converter diff --git a/docs/converters/smac.rst b/docs/converters/smac.rst index e6f75eda..7d0e7e92 100644 --- a/docs/converters/smac.rst +++ b/docs/converters/smac.rst @@ -7,7 +7,7 @@ runs created with the newer SMAC version >= 2.0.0. As Auto-Sklearn and Auto-PyTorch rely on SMAC for hyperparameter optimization, the SMAC converter also allows to load runs created with these tools. -To be able to load it correctly into DeepCAVE, you need save the configuration space as a ConfigSpace JSON file, +To be able to load it correctly into DeepCAVE, you need to save the configuration space as a ConfigSpace JSON file, as well as the run history: .. code-block:: python