Parallel runs for Optuna

examples/optimization/conditional_parameters.py

@@ -18,6 +18,7 @@
 train_data, test_data = train_test_split(data, test_size=0.2, stratify=data["TARGET"], random_state=42)
 
 
+# replacing default _sample function in OptunaTuner class with this function
 def sample(optimization_search_space, trial, suggested_params):
     trial_values = copy.copy(suggested_params)
     trial_values["feature_fraction"] = trial.suggest_uniform("feature_fraction", low=0.5, high=1.0)

examples/optimization/custom_search_space.py

@@ -8,9 +8,8 @@
 from sklearn.model_selection import train_test_split
 
 from lightautoml.automl.presets.tabular_presets import TabularAutoML
-from lightautoml.ml_algo.tuning.base import Distribution
-from lightautoml.ml_algo.tuning.base import SearchSpace
 from lightautoml.tasks import Task
+from lightautoml.ml_algo.tuning.base import Uniform
 
 
 # load and prepare data
@@ -22,8 +21,8 @@
     task=Task("binary"),
     lgb_params={
         "optimization_search_space": {
-            "feature_fraction": SearchSpace(Distribution.UNIFORM, low=0.5, high=1.0),
-            "min_sum_hessian_in_leaf": SearchSpace(Distribution.LOGUNIFORM, low=1e-3, high=10.0),
+            "feature_fraction": Uniform(low=0.5, high=1.0),
+            "min_sum_hessian_in_leaf": Uniform(low=1e-3, high=10.0, log=True),
         }
     },
 )

examples/optimization/sequential_parameter_search.py

@@ -12,13 +12,12 @@
 from lightautoml.automl.presets.tabular_presets import TabularAutoML
 from lightautoml.tasks import Task
 
-
 # load and prepare data
 data = pd.read_csv("./data/sampled_app_train.csv")
 train_data, test_data = train_test_split(data, test_size=0.2, stratify=data["TARGET"], random_state=42)
 
 
-def sample(optimization_search_space, trial, suggested_params):
+def sample(trial, suggested_params):
     trial_values = copy.copy(suggested_params)
 
     for feature_fraction in range(10):

lightautoml/automl/presets/tabular_presets.py

@@ -30,7 +30,6 @@
 from ...ml_algo.dl_model import TorchModel
 from ...ml_algo.linear_sklearn import LinearLBFGS
 from ...ml_algo.random_forest import RandomForestSklearn
-from ...ml_algo.tuning.optuna import DLOptunaTuner
 from ...ml_algo.tuning.optuna import OptunaTuner
 from ...pipelines.features.lgb_pipeline import LGBAdvancedPipeline
 from ...pipelines.features.lgb_pipeline import LGBSeqSimpleFeatures
@@ -444,7 +443,7 @@ def get_nn(
 
             if tuned:
                 nn_model.set_prefix("Tuned")
-                nn_tuner = DLOptunaTuner(
+                nn_tuner = OptunaTuner(
                     n_trials=model_params["tuning_params"]["max_tuning_iter"],
                     timeout=model_params["tuning_params"]["max_tuning_time"],
                     fit_on_holdout=model_params["tuning_params"]["fit_on_holdout"],

lightautoml/automl/presets/text_presets.py

@@ -22,7 +22,6 @@
 from ...ml_algo.boost_lgbm import BoostLGBM
 from ...ml_algo.dl_model import TorchModel
 from ...ml_algo.linear_sklearn import LinearLBFGS
-from ...ml_algo.tuning.optuna import DLOptunaTuner
 from ...ml_algo.tuning.optuna import OptunaTuner
 from ...pipelines.features.base import FeaturesPipeline
 from ...pipelines.features.lgb_pipeline import LGBAdvancedPipeline
@@ -307,7 +306,7 @@ def get_nn(
 
             if tuned:
                 nn_model.set_prefix("Tuned")
-                nn_tuner = DLOptunaTuner(
+                nn_tuner = OptunaTuner(
                     n_trials=model_params["tuning_params"]["max_tuning_iter"],
                     timeout=model_params["tuning_params"]["max_tuning_time"],
                     fit_on_holdout=model_params["tuning_params"]["fit_on_holdout"],

lightautoml/ml_algo/base.py

@@ -8,6 +8,7 @@
 from typing import Any
 from typing import Dict
 from typing import List
+from typing import Callable
 from typing import Optional
 from typing import Sequence
 from typing import Tuple
@@ -48,7 +49,13 @@ class MLAlgo(ABC):
     """
 
     _default_params: Dict = {}
-    optimization_search_space: Dict = {}
+
+    # Dict is a default search space representation that is used for simple cases
+    # Callable is used for complex cases like conditional search space as described in
+    # LightAutoML/examples/optimization/conditional_parameters.py
+    # Called in _get_objective function in OptunaTuner class
+    optimization_search_space: Union[Dict, Callable] = {}
+
     # TODO: add checks here
     _fit_checks: Tuple = ()
     _transform_checks: Tuple = ()

lightautoml/ml_algo/dl_model.py

@@ -17,13 +17,13 @@
 from typing import Dict
 
 import numpy as np
-import optuna
 import pandas as pd
 import torch
 import torch.nn as nn
 
 from torch.optim.lr_scheduler import ReduceLROnPlateau
 
+from .tuning.base import Uniform
 from ..dataset.np_pd_dataset import NumpyDataset
 from ..tasks.losses.torch import TorchLossWrapper
 from ..utils.installation import __validate_extra_deps
@@ -618,7 +618,7 @@ def predict_single_fold(self, model: any, dataset: TabularDataset) -> np.ndarray
 
         return pred
 
-    def _default_sample(self, trial: optuna.trial.Trial, estimated_n_trials: int, suggested_params: Dict) -> Dict:
+    def _get_default_search_spaces(self, estimated_n_trials: int, suggested_params: Dict) -> Dict:
         """Implements simple tuning sampling strategy.
 
         Args:
@@ -631,19 +631,18 @@ def _default_sample(self, trial: optuna.trial.Trial, estimated_n_trials: int, su
 
         """
         # optionally
-        trial_values = copy(suggested_params)
+        # trial_values = copy(suggested_params) # TODO: check how to use it
+        trial_values = {}
 
-        trial_values["bs"] = trial.suggest_categorical("bs", [2 ** i for i in range(6, 11)])
+        trial_values["bs"] = Uniform(low=64, high=1024, q=1)
 
-        weight_decay_bin = trial.suggest_categorical("weight_decay_bin", [0, 1])
+        weight_decay_bin = Uniform(low=0, high=1, q=1)
         if weight_decay_bin == 0:
             weight_decay = 0
         else:
-            weight_decay = trial.suggest_loguniform("weight_decay", low=1e-6, high=1e-2)
+            weight_decay = Uniform(low=1e-6, high=1e-2, log=True)
+
+        trial_values["lr"] = Uniform(low=1e-5, high=1e-1, log=True)
+        trial_values["weight_decay"] = weight_decay
 
-        lr = trial.suggest_loguniform("lr", low=1e-5, high=1e-1)
-        trial_values["opt_params"] = {
-            "lr": lr,
-            "weight_decay": weight_decay,
-        }
         return trial_values