diff --git a/README.md b/README.md index 33fb8da..ba98e2e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # keras-hypetune -A friendly python package for Keras Hyperparameters Tuning based only on NumPy. +A friendly python package for Keras Hyperparameters Tuning based only on NumPy and Hyperopt. ## Overview @@ -71,6 +71,27 @@ krs = KerasRandomSearch(get_model, param_grid, monitor='val_loss', greater_is_be krs.search(x_train, y_train, validation_data=(x_valid, y_valid)) ``` +### KerasBayesianSearch + +The parameter values are chosen according to bayesian optimization algorithms based on gaussian processes and regression trees (from hyperopt). + +The number of parameter combinations that are tried is given by n_iter. Parameters must be given as hyperopt distributions. + +```python +param_grid = { + 'unit_1': 64 + hp.randint('unit_1', 64), + 'unit_2': 32 + hp.randint('unit_2', 96), + 'lr': hp.loguniform('lr', np.log(0.001), np.log(0.02)), + 'activ': hp.choice('activ', ['elu','relu']), + 'epochs': 100, + 'batch_size': 512 +} + +kbs = KerasBayesianSearch(get_model, param_grid, monitor='val_loss', greater_is_better=False, + n_iter=15, sampling_seed=33) +kbs.search(x_train, y_train, trials=Trials(), validation_data=(x_valid, y_valid)) +``` + ## Cross Validation This tuning modality operates the optimization using a cross-validation approach. The CV strategies available are the same provided by scikit-learn splitter classes. The parameter combinations are evaluated on the mean score of the folds. In this case, it's allowed the usage of only numpy array data. For tasks involving multi-input/output, the arrays can be wrapped into list or dict like in normal Keras. @@ -117,3 +138,26 @@ krs = KerasRandomSearchCV(get_model, param_grid, cv=cv, monitor='val_loss', grea n_iter=15, sampling_seed=33) krs.search(X, y) ``` + +### KerasBayesianSearchCV + +The parameter values are chosen according to bayesian optimization algorithms based on gaussian processes and regression trees (from hyperopt). + +The number of parameter combinations that are tried is given by n_iter. Parameters must be given as hyperopt distributions. + +```python +param_grid = { + 'unit_1': 64 + hp.randint('unit_1', 64), + 'unit_2': 32 + hp.randint('unit_2', 96), + 'lr': hp.loguniform('lr', np.log(0.001), np.log(0.02)), + 'activ': hp.choice('activ', ['elu','relu']), + 'epochs': 100, + 'batch_size': 512 +} + +cv = KFold(n_splits=3, random_state=33, shuffle=True) + +kbs = KerasBayesianSearchCV(get_model, param_grid, cv=cv, monitor='val_loss', greater_is_better=False, + n_iter=15, sampling_seed=33) +kbs.search(X, y, trials=Trials()) +``` diff --git a/kerashypetune/__init__.py b/kerashypetune/__init__.py index 896380c..5956d87 100644 --- a/kerashypetune/__init__.py +++ b/kerashypetune/__init__.py @@ -1,2 +1,3 @@ from .utils import * +from ._classes import * from .kerashypetune import * \ No newline at end of file diff --git a/kerashypetune/_classes.py b/kerashypetune/_classes.py new file mode 100644 index 0000000..8ccca54 --- /dev/null +++ b/kerashypetune/_classes.py @@ -0,0 +1,346 @@ +import inspect +import numpy as np +from copy import deepcopy + +from hyperopt import fmin, tpe + +from .utils import (ParameterSampler, _check_param, _check_data, + _clear_callbacks, _create_fold, _is_multioutput) + + +class _KerasSearch: + """Base class for KerasSearch meta-estimator. + + Warning: This class should not be used directly. Use derived classes + instead. + """ + + def __init__(self, + hypermodel, + param_grid, + n_iter=None, + sampling_seed=None, + monitor='val_loss', + greater_is_better=False, + store_model=True, + savepath=None, + tuner_verbose=1): + + self.hypermodel = hypermodel + self.param_grid = param_grid + self.n_iter = n_iter + self.sampling_seed = sampling_seed + self.monitor = monitor + self.greater_is_better = greater_is_better + self.store_model = store_model + self.savepath = savepath + self.tuner_verbose = tuner_verbose + + def _fit(self, + x, y, + params, + callbacks, + validation_split, + validation_data, + id_fold, fitargs): + """Private method to fit a Keras model.""" + + if not callable(self.hypermodel): + raise ValueError("hypermodel must be a callable function.") + + results = {} + tunable_fitargs = ['batch_size', 'epochs', + 'steps_per_epoch', 'class_weight'] + + if callbacks is not None: + fitargs['callbacks'] = _clear_callbacks( + deepcopy(callbacks), self._trial_id, id_fold) + + model = self.hypermodel(params) + fit_params = {k: v for k, v in params.items() if k in tunable_fitargs} + all_fitargs = dict(list(fitargs.items()) + list(fit_params.items())) + + if self.tuner_verbose > 0: + print( + "\n***** ({}/{}) *****\nSearch({})".format( + self._trial_id, + self.n_iter if self._tuning_type is 'hyperopt' \ + else len(self._param_combi), + params + ) + ) + + model.fit(x=x, + y=y, + validation_split=validation_split, + validation_data=validation_data, + **all_fitargs) + + epoch = self._eval_score(model.history.history[self.monitor]) + params['epochs'] = epoch + 1 + params['steps_per_epoch'] = model.history.params['steps'] + params['batch_size'] = (all_fitargs['batch_size'] if 'batch_size' + in all_fitargs else None) + + score = round(model.history.history[self.monitor][epoch], 5) + + if self.tuner_verbose > 0: + print("SCORE: {} at epoch {}".format(score, epoch + 1)) + + results = { + 'params': params, 'status': 'ok', + 'loss': score * self._score_sign, + 'model': model + } + self._trial_id += 1 + + return results + + def _search(self, + tuning_type, + x, y=None, + trials=None, + validation_data=None, + validation_split=0.0, + id_fold=None, + **fitargs): + """Private method to perform a search on a fixed validation set for + the best parameters configuration.""" + + if validation_data is None and validation_split == 0.0: + raise ValueError( + "Pass at least one of validation_data or validation_split." + ) + + if not isinstance(self.param_grid, dict): + raise ValueError("Pass param_grid in dict format.") + self._param_grid = self.param_grid.copy() + + for p_k, p_v in self._param_grid.items(): + self._param_grid[p_k] = _check_param(p_v) + + self._eval_score = np.argmax if self.greater_is_better else np.argmin + self._score_sign = -1 if self.greater_is_better else 1 + + rs = ParameterSampler(n_iter=self.n_iter, + param_distributions=self._param_grid, + random_state=self.sampling_seed) + self._param_combi, self._tuning_type = rs.sample() + self._trial_id = 1 + + tuning_class = { + 'grid': 'KerasGridSearch', + 'random': 'KerasRandomSearch', + 'hyperopt': 'KerasBayesianSearch' + } + if tuning_type != self._tuning_type: + raise ValueError( + "The chosen param_grid is incompatible with {}. " + "Maybe you are looking for {}.".format( + tuning_class[tuning_type] + \ + ('CV' if id_fold is not None else ''), + tuning_class[self._tuning_type] + \ + ('CV' if id_fold is not None else '') + ) + ) + + if 'callbacks' in fitargs: + if isinstance(fitargs['callbacks'], list): + callbacks = deepcopy(fitargs['callbacks']) + else: + callbacks = deepcopy([fitargs['callbacks']]) + else: + callbacks = None + + if self.tuner_verbose > 0: + n_trials = self.n_iter if self._tuning_type is 'hyperopt' \ + else len(self._param_combi) + print("\n{} trials detected for {}".format( + n_trials, tuple(self._param_grid.keys()))) + verbose = fitargs['verbose'] if 'verbose' in fitargs else 0 + else: + verbose = 0 + fitargs['verbose'] = verbose + + if self._tuning_type == 'hyperopt': + if trials is None: + raise ValueError( + "trials must be not None when using hyperopt." + ) + + search = fmin( + fn=lambda p: self._fit( + params=p, x=x, y=y, + callbacks=callbacks, + validation_split=validation_split, + validation_data=validation_data, + id_fold=id_fold, fitargs=fitargs + ), + space=self._param_combi, algo=tpe.suggest, + max_evals=self.n_iter, trials=trials, + rstate=np.random.RandomState(self.sampling_seed), + show_progressbar=False, verbose=0 + ) + all_results = trials.results + + else: + all_results = [ + self._fit( + params=params, x=x, y=y, + callbacks=callbacks, + validation_split=validation_split, + validation_data=validation_data, + id_fold=id_fold, fitargs=fitargs + ) + for params in self._param_combi + ] + + self.trials, self.scores, models = [], [], [] + for res in all_results: + self.trials.append(res['params']) + self.scores.append(self._score_sign * res['loss']) + models.append(res['model']) + + # get the best + id_best = self._eval_score(self.scores) + self.best_score = self.scores[id_best] + self.best_params = self.trials[id_best] + best_model = models[id_best] + if self.store_model: + self.best_model = best_model + + if self.savepath is not None: + if id_fold is not None: + best_model.save(self.savepath.replace('{fold}', str(id_fold))) + else: + best_model.save(self.savepath) + + return self + + +class _KerasSearchCV: + """Base class for KerasSearchCV meta-estimator. + + Warning: This class should not be used directly. Use derived classes + instead. + """ + + def __init__(self, + hypermodel, + param_grid, + cv, + n_iter=None, + sampling_seed=None, + monitor='val_loss', + greater_is_better=False, + store_model=True, + savepath=None, + tuner_verbose=1): + + self.hypermodel = hypermodel + self.param_grid = param_grid + self.cv = cv + self.n_iter = n_iter + self.sampling_seed = sampling_seed + self.monitor = monitor + self.greater_is_better = greater_is_better + self.store_model = store_model + self.savepath = savepath + self.tuner_verbose = tuner_verbose + + def _search(self, + tuning_type, + x, y=None, + trials=None, + sample_weight=None, + groups=None, + **fitargs): + """Private method to perform a CV search for the best parameters + configuration.""" + + self.folds_trials = {} + self.folds_scores = {} + self.folds_best_params = {} + self.folds_best_score = {} + if self.store_model: + self.folds_best_models = {} + + if 'validation_split' in fitargs or 'validation_data' in fitargs: + raise ValueError( + "Validation is automatically created by the cv strategy.") + + if not hasattr(self.cv, 'split'): + raise ValueError( + "Expected cv as cross-validation object with split method to " + "generate indices to split data into training and test set " + "(like from sklearn.model_selection).") + else: + split_args = inspect.signature(self.cv.split).parameters + split_args = {k: v.default for k, v in split_args.items()} + split_need_y = split_args['y'] is not None + + _x = _check_data(x) + + if y is not None: + _y = _check_data(y, is_target=True) + if _is_multioutput(y) and split_need_y: + raise ValueError( + "{} not supports multioutput.".format(self.cv)) + else: + _y = None + + if sample_weight is not None: + _ = _check_data(sample_weight) + + for fold, (train_id, val_id) in enumerate(self.cv.split(_x, _y, groups)): + + if self.tuner_verbose > 0: + print("\n{}\n{} Fold {} {}\n{}".format( + '#' * 18, '#' * 3, str(fold + 1).zfill(3), '#' * 3, '#' * 18)) + + x_train = _create_fold(x, train_id) + y_train = None if y is None else _create_fold(y, train_id) + sample_weight_train = (_create_fold(sample_weight, train_id) + if sample_weight is not None else None) + + x_val = _create_fold(x, val_id) + y_val = None if y is None else _create_fold(y, val_id) + sample_weight_val = (_create_fold(sample_weight, val_id) + if sample_weight is not None else None) + + ks_fold = _KerasSearch(hypermodel=self.hypermodel, + param_grid=self.param_grid, + n_iter=self.n_iter, + sampling_seed=self.sampling_seed, + monitor=self.monitor, + greater_is_better=self.greater_is_better, + store_model=self.store_model, + savepath=self.savepath, + tuner_verbose=self.tuner_verbose) + + _trials = trials if trials is None else deepcopy(trials) + ks_fold._search(tuning_type=tuning_type, + x=x_train, y=y_train, + trials=_trials, + sample_weight=sample_weight_train, + validation_data=(x_val, y_val, sample_weight_val), + id_fold=fold + 1, + **fitargs) + + fold_id = "fold {}".format(fold + 1) + self.folds_trials[fold_id] = ks_fold.trials + self.folds_scores[fold_id] = ks_fold.scores + self.folds_best_params[fold_id] = ks_fold.best_params + if self.store_model: + self.folds_best_models[fold_id] = ks_fold.best_model + self.folds_best_score[fold_id] = ks_fold.best_score + + eval_score = np.argmax if self.greater_is_better else np.argmin + mean_score_params = np.mean( + list(self.folds_scores.values()), axis=0).round(5) + evaluate = eval_score(mean_score_params) + + self.best_params = [list(f)[evaluate] for f in self.folds_trials.values()] + self.best_params_score = mean_score_params[evaluate] + + return self \ No newline at end of file diff --git a/kerashypetune/kerashypetune.py b/kerashypetune/kerashypetune.py index 9c8a8d5..a62caa7 100644 --- a/kerashypetune/kerashypetune.py +++ b/kerashypetune/kerashypetune.py @@ -1,149 +1,4 @@ -import random -import inspect -import numpy as np -from copy import deepcopy - -from .utils import (ParameterSampler, _check_param, _check_data, - _clear_callbacks, _create_fold, _is_multioutput) - - -class _KerasSearch: - """Base class for KerasSearch meta-estimator. - - Warning: This class should not be used directly. Use derived classes - instead. - """ - - def __init__(self, - hypermodel, - param_grid, - n_iter=None, - sampling_seed=None, - monitor='val_loss', - greater_is_better=False, - store_model=True, - savepath=None, - tuner_verbose=1): - - self.hypermodel = hypermodel - self.param_grid = param_grid - self.n_iter = n_iter - self.sampling_seed = sampling_seed - self.monitor = monitor - self.greater_is_better = greater_is_better - self.store_model = store_model - self.savepath = savepath - self.tuner_verbose = tuner_verbose - - def __repr__(self): - return "".format(self.__class__.__name__) - - def __str__(self): - return "".format(self.__class__.__name__) - - def _search(self, - x, y=None, - validation_data=None, - validation_split=0.0, - is_random=False, - id_fold=None, - **fitargs): - """Private method to perform a search on a fixed validation set for - the best parameters configuration.""" - - self.trials = [] - self.scores = [] - - if validation_data is None and validation_split == 0.0: - raise ValueError( - "Pass at least one of validation_data or validation_split.") - - if not isinstance(self.param_grid, dict): - raise ValueError("Pass param_grid in dict format.") - self._param_grid = self.param_grid.copy() - - tunable_fitargs = ['batch_size', 'epochs', - 'steps_per_epoch', 'class_weight'] - - for p_k, p_v in self._param_grid.items(): - self._param_grid[p_k] = _check_param(p_v) - - eval_epoch = np.argmax if self.greater_is_better else np.argmin - eval_score = np.max if self.greater_is_better else np.min - start_score = -np.inf if self.greater_is_better else np.inf - self.best_score = start_score - - rs = ParameterSampler(n_iter=self.n_iter, - param_distributions=self._param_grid, - random_state=self.sampling_seed, - is_random=is_random) - self._param_combi = rs.sample() - - if 'callbacks' in fitargs: - if isinstance(fitargs['callbacks'], list): - _callbacks = deepcopy(fitargs['callbacks']) - else: - _callbacks = deepcopy([fitargs['callbacks']]) - - if self.tuner_verbose > 0: - print("\n{} trials detected for {}".format( - len(self._param_combi), tuple(self._param_grid.keys()))) - verbose = fitargs['verbose'] if 'verbose' in fitargs else 0 - else: - verbose = 0 - fitargs['verbose'] = verbose - - for trial, param in enumerate(self._param_combi): - - if 'callbacks' in fitargs: - fitargs['callbacks'] = _clear_callbacks( - deepcopy(_callbacks), trial + 1, id_fold) - - param = dict(zip(self._param_grid.keys(), param)) - model = self.hypermodel(param) - - fit_param = {k: v for k, v in param.items() if k in tunable_fitargs} - all_fitargs = dict(list(fitargs.items()) + list(fit_param.items())) - - if self.tuner_verbose > 0: - print("\n***** ({}/{}) *****\nSearch({})".format( - trial + 1, len(self._param_combi), param)) - - model.fit(x=x, - y=y, - validation_split=validation_split, - validation_data=validation_data, - **all_fitargs) - - epoch = eval_epoch(model.history.history[self.monitor]) - param['epochs'] = epoch + 1 - param['steps_per_epoch'] = model.history.params['steps'] - param['batch_size'] = (all_fitargs['batch_size'] if 'batch_size' - in all_fitargs else None) - score = round(model.history.history[self.monitor][epoch], 5) - evaluate = eval_score([self.best_score, score]) - - if self.best_score != evaluate: - - self.best_params = param - - if self.store_model: - self.best_model = model - - if self.savepath is not None: - if id_fold is not None: - model.save(self.savepath.replace('{fold}', str(id_fold))) - else: - model.save(self.savepath) - - self.best_score = evaluate - self.trials.append(param) - self.scores.append(score) - - if self.tuner_verbose > 0: - print("SCORE: {} at epoch {}".format(score, epoch + 1)) - - return self +from ._classes import _KerasSearch, _KerasSearchCV class KerasGridSearch(_KerasSearch): @@ -227,7 +82,6 @@ def __init__(self, store_model=True, savepath=None, tuner_verbose=1): - self.hypermodel = hypermodel self.param_grid = param_grid self.monitor = monitor @@ -273,10 +127,10 @@ def search(self, self : object """ - self._search(x=x, y=y, + self._search(tuning_type='grid', + x=x, y=y, validation_data=validation_data, validation_split=validation_split, - is_random=False, **fitargs) return self @@ -360,7 +214,7 @@ class KerasRandomSearch(_KerasSearch): Notes ---------- - KerasGridSearch allows the usage of every callbacks available in Keras + KerasRandomSearch allows the usage of every callbacks available in Keras (also the custom one). The callbacks, that provide the possibility to save any output as external files, support naming formatting options. This is true for ModelCheckpoint, CSVLogger, TensorBoard and RemoteMonitor. @@ -382,7 +236,6 @@ def __init__(self, store_model=True, savepath=None, tuner_verbose=1): - self.hypermodel = hypermodel self.param_grid = param_grid self.n_iter = n_iter @@ -428,37 +281,115 @@ def search(self, self : object """ - self._search(x=x, y=y, + self._search(tuning_type='random', + x=x, y=y, validation_data=validation_data, validation_split=validation_split, - is_random=True, **fitargs) return self -class _KerasSearchCV: - """Base class for KerasSearchCV meta-estimator. +class KerasBayesianSearch(_KerasSearch): + """Bayesian hyperparamater searching and optimization on a fixed + validation set. + + Pass a Keras model (in Sequential or Functional format), and + a dictionary with the parameter boundaries for the experiment. + For searching, takes in the same arguments available in Keras + model.fit(...). All the input format supported by Keras model + are accepted. + + In contrast to random-search, the parameter settings are not sampled + randomly. The parameter values are chosen according to bayesian + optimization algorithms based on gaussian processes and regression trees. + The number of parameter settings that are tried is given by n_iter. + Parameters must be given as hyperopt distributions. + It is highly recommended to use continuous distributions for continuous + parameters. - Warning: This class should not be used directly. Use derived classes - instead. + Parameters + ---------- + hypermodel : callable + A callable that takes parameters in dict format and returns a + TF Model instance. + + param_grid : dict + Hyperparameters to try, 1-to-1 mapped with the parameters dict + keys present in the hypermodel function. + + n_iter : int + Number of parameter settings that are sampled. + n_iter trades off runtime vs quality of the solution. + + sampling_seed : int, default=0 + The seed used to sample from the hyperparameter distributions. + + monitor : str, default='val_loss' + Quantity to monitor in order to detect the best model. + + greater_is_better : bool, default=False + Whether the quantity to monitor is a score function, meaning high + is good, or a loss function (as default), meaning low is good. + + store_model : bool, default=True + If True the best model is stored in the KerasGridSearch object. + + savepath : str, default=None + String or path-like, path to save the best model file. + If None, no saving is applied. + + tuner_verbose : int, default=1 + Verbosity mode. <=0 silent all; >0 print trial logs with the + connected score. + + Attributes + ---------- + trials : list + A list of dicts. The dicts are all the hyperparameter combinations + tried and derived from the param_grid. + + scores : list + The monitor quantities achived on the validation data by all the + models tried. + + best_params : dict + The dict containing the best combination (in term of score) of + hyperparameters. + + best_score : float + The best score achieved by all the possible combination created. + + best_model : TF Model + The best model (in term of score). Accessible only if store_model + is set to True. + + Notes + ---------- + KerasBayesianSearch allows the usage of every callbacks available in Keras + (also the custom one). The callbacks, that provide the possibility to + save any output as external files, support naming formatting options. + This is true for ModelCheckpoint, CSVLogger, TensorBoard and RemoteMonitor. + 'trial' is the custom token that can be used to personalize name formatting. + + For example: if filepath in ModelCheckpoint is model_{trial}.hdf5, then + the model checkpoints will be saved with the relative number of trial in + the filename. This enables to save and differentiate each model created in + the searching trials. """ def __init__(self, hypermodel, param_grid, - cv, - n_iter=None, - sampling_seed=None, + n_iter, + sampling_seed=0, monitor='val_loss', greater_is_better=False, store_model=True, savepath=None, tuner_verbose=1): - self.hypermodel = hypermodel self.param_grid = param_grid - self.cv = cv self.n_iter = n_iter self.sampling_seed = sampling_seed self.monitor = monitor @@ -467,110 +398,61 @@ def __init__(self, self.savepath = savepath self.tuner_verbose = tuner_verbose - def __repr__(self): - return "".format(self.__class__.__name__) - - def __str__(self): - return "".format(self.__class__.__name__) - - def _search(self, - x, y=None, - sample_weight=None, - groups=None, - is_random=False, - **fitargs): - """Private method to perform a CV search for the best parameters - configuration.""" - - self.folds_trials = {} - self.folds_scores = {} - self.folds_best_params = {} - self.folds_best_score = {} - if self.store_model: - self.folds_best_models = {} - - if 'validation_split' in fitargs or 'validation_data' in fitargs: - raise ValueError( - "Validation is automatically created by the cv strategy.") - - if not hasattr(self.cv, 'split'): - raise ValueError( - "Expected cv as cross-validation object with split method to " - "generate indices to split data into training and test set " - "(like from sklearn.model_selection).") - else: - split_args = inspect.signature(self.cv.split).parameters - split_args = {k: v.default for k, v in split_args.items()} - split_need_y = split_args['y'] is not None - - _x = _check_data(x) - - if y is not None: - _y = _check_data(y, is_target=True) - if _is_multioutput(y) and split_need_y: - raise ValueError( - "{} not supports multioutput.".format(self.cv)) - else: - _y = None - - if sample_weight is not None: - _ = _check_data(sample_weight) - - for fold, (train_id, val_id) in enumerate(self.cv.split(_x, _y, groups)): - - if self.tuner_verbose > 0: - print("\n{}\n{} Fold {} {}\n{}".format( - '#' * 18, '#' * 3, str(fold + 1).zfill(3), '#' * 3, '#' * 18)) - - x_train = _create_fold(x, train_id) - y_train = None if y is None else _create_fold(y, train_id) - sample_weight_train = (_create_fold(sample_weight, train_id) - if sample_weight is not None else None) - - x_val = _create_fold(x, val_id) - y_val = None if y is None else _create_fold(y, val_id) - sample_weight_val = (_create_fold(sample_weight, val_id) - if sample_weight is not None else None) - - ks_fold = _KerasSearch(hypermodel=self.hypermodel, - param_grid=self.param_grid, - n_iter=self.n_iter, - sampling_seed=self.sampling_seed, - monitor=self.monitor, - greater_is_better=self.greater_is_better, - store_model=self.store_model, - savepath=self.savepath, - tuner_verbose=self.tuner_verbose) - - ks_fold._search(x=x_train, y=y_train, - sample_weight=sample_weight_train, - validation_data=(x_val, y_val, sample_weight_val), - is_random=is_random, - id_fold=fold + 1, - **fitargs) - - fold_id = "fold {}".format(fold + 1) - self.folds_trials[fold_id] = ks_fold.trials - self.folds_scores[fold_id] = ks_fold.scores - self.folds_best_params[fold_id] = ks_fold.best_params - if self.store_model: - self.folds_best_models[fold_id] = ks_fold.best_model - self.folds_best_score[fold_id] = ks_fold.best_score - - eval_score = np.argmax if self.greater_is_better else np.argmin - mean_score_params = np.mean( - list(self.folds_scores.values()), axis=0).round(5) - evaluate = eval_score(mean_score_params) - - self.best_params = [list(f)[evaluate] for f in self.folds_trials.values()] - self.best_params_score = mean_score_params[evaluate] + def search(self, + x, y=None, + trials=None, + validation_data=None, + validation_split=0.0, + **fitargs): + """Performs a search for best hyperparameter configurations creating + all the possible trials and evaluating on the validation set provided. + + Parameters + ---------- + x : multi types + Input data. All the input formats supported by Keras model are + accepted. + + y : multi types, default=None + Target data. All the target formats supported by Keras model are + accepted. + + trials : hyperopt.Trials() object, default=None + A hyperopt trials object, used to store intermediate results for all + optimization runs. Effective (and required) only when hyperopt + parameter searching is computed. + + validation_data : multi types, default=None + Data on which evaluate the loss and any model metrics at the end of + each epoch. All the validation_data formats supported by Keras model + are accepted. + + validation_split : float, default=0.0 + Float between 0 and 1. Fraction of the training data to be used as + validation data. + + **fitargs : Additional fitting arguments, the same accepted in Keras + model.fit(...). + + Returns + ------- + self : object + """ + + self._search(tuning_type='hyperopt', + x=x, y=y, + trials=trials, + validation_data=validation_data, + validation_split=validation_split, + **fitargs) return self class KerasGridSearchCV(_KerasSearchCV): - """Grid hyperparamater searching and optimization on a fixed - validation set. + """Grid hyperparamater searching and optimization with cross + validation. Out-of-fold samples generated by CV are used automatically + as validation_data in each model fitting. Pass a Keras model (in Sequential or Functional format), and a dictionary with the parameter boundaries for the experiment. @@ -643,7 +525,6 @@ class KerasGridSearchCV(_KerasSearchCV): The paramareter combination related to the best average score in all the available folds. - Notes ---------- KerasGridSearchCV allows the usage of every callbacks available in Keras @@ -668,7 +549,6 @@ def __init__(self, store_model=True, savepath=None, tuner_verbose=1): - self.hypermodel = hypermodel self.param_grid = param_grid self.cv = cv @@ -718,18 +598,19 @@ def search(self, self : object """ - self._search(x=x, y=y, + self._search(tuning_type='grid', + x=x, y=y, sample_weight=sample_weight, groups=groups, - is_random=False, **fitargs) return self class KerasRandomSearchCV(_KerasSearchCV): - """Random hyperparamater searching and optimization on a fixed - validation set. + """Random hyperparamater searching and optimization with cross + validation. Out-of-fold samples generated by CV are used automatically + as validation_data in each model fitting. Pass a Keras model (in Sequential or Functional format), and a dictionary with the parameter boundaries for the experiment. @@ -845,7 +726,6 @@ def __init__(self, store_model=True, savepath=None, tuner_verbose=1): - self.hypermodel = hypermodel self.param_grid = param_grid self.cv = cv @@ -895,10 +775,192 @@ def search(self, self : object """ - self._search(x=x, y=y, + self._search(tuning_type='random', + x=x, y=y, sample_weight=sample_weight, groups=groups, - is_random=True, **fitargs) return self + + +class KerasBayesianSearchCV(_KerasSearchCV): + """Bayesian hyperparamater searching and optimization with cross + validation. Out-of-fold samples generated by CV are used automatically + as validation_data in each model fitting. + + Pass a Keras model (in Sequential or Functional format), and + a dictionary with the parameter boundaries for the experiment. + For searching, takes in the same arguments available in Keras + model.fit(...). The cross-validation strategies are the same + provided by the scikit-learn cross-validation generator. + Only input in array format are supported. In case of multi-input or + multi-output is it possible to wrap arrays in list or dictionaries + like in Keras. + + In contrast to random-search, the parameter settings are not sampled + randomly. The parameters values are chosen according to bayesian + optimization algorithms based on gaussian processes and regression trees. + The number of parameter settings that are tried is given by n_iter. + Parameters must be given as hyperopt distributions. + It is highly recommended to use continuous distributions for continuous + parameters. + + Parameters + ---------- + hypermodel : callable + A callable that takes parameters in dict format and returns a + TF Model instance. + + param_grid : dict + Hyperparameters to try, 1-to-1 mapped with the parameters dict + keys present in the hypermodel function. + + cv : scikit-learn cross-validation generator + An sklearn.model_selection splitter class. Used to determine + how samples are split up into groups for cross-validation. + + n_iter : int + Number of parameter settings that are sampled. + n_iter trades off runtime vs quality of the solution. + + sampling_seed : int, default=0 + The seed used to sample from the hyperparameter distributions. + + monitor : str, default='val_loss' + Quantity to monitor in order to detect the best model. + + greater_is_better : bool, default=False + Whether the quantity to monitor is a score function, meaning high + is good, or a loss function (as default), meaning low is good. + + store_model : bool, default=True + If True the best model of each fold is stored inside the + KerasRandomSearchCV object. + + savepath : str, default=None + String or path-like, path to save the best model file. + If None, no saving is applied. + + tuner_verbose : int, default=1 + Verbosity mode. <=0 silent all; >0 print trial logs with the + connected score. + + Attributes + ---------- + folds_trials : dict + A dicts of list. The lists contain all the hyperparameter combinations + tried in each fold and derived from the param_grid. + + folds_scores : dict + A dicts of list. The lists contain the monitor quantities achived on + the validation data by all the models tried in each fold. + + folds_best_params : dict + The dict containing the best combination (in term of score) of + hyperparameters in each fold. + + folds_best_score : dict + The best scores achieved by all the possible combination created in + each fold. + + folds_best_model : dict + The best models (in term of score) in each fold. Accessible only if + store_model is set to True. + + best_params_score : float + The best average score in all the available folds. + + best_params : dict + The paramareter combination related to the best average score in all + the available folds. + + Notes + ---------- + KerasBayesianSearchCV allows the usage of every callbacks available in Keras + (also the custom one). The callbacks, that provide the possibility to + save any output as external files, support naming formatting options. + This is true for ModelCheckpoint, CSVLogger, TensorBoard and RemoteMonitor. + 'trial' and 'fold' are custom tokens that can be used to personalize the + name formatting. + + For example: if filepath in ModelCheckpoint is model_{fold}_{trial}.hdf5, + then the model checkpoints will be saved with the relative number of trial, + obtained at a certain fold, in the filename. This enables to save and + differentiate each model created in the searching trials. + """ + + def __init__(self, + hypermodel, + param_grid, + cv, + n_iter, + sampling_seed=0, + monitor='val_loss', + greater_is_better=False, + store_model=True, + savepath=None, + tuner_verbose=1): + self.hypermodel = hypermodel + self.param_grid = param_grid + self.cv = cv + self.n_iter = n_iter + self.sampling_seed = sampling_seed + self.monitor = monitor + self.greater_is_better = greater_is_better + self.store_model = store_model + self.savepath = savepath + self.tuner_verbose = tuner_verbose + + def search(self, + x, y=None, + trials=None, + sample_weight=None, + groups=None, + **fitargs): + """Performs a search for best hyperparameter configurations creating + all the possible trials and evaluating on the validation folds + created following the validation strategy. + + Parameters + ---------- + x : multi types + Input data. Accepted types are arrays or list/dict in case of + multi-input/output. + + y : multi types, default=None + Target data. Accepted types are arrays or list/dict in case of + multi-input/output. + + trials : hyperopt.Trials() object, default=None + A hyperopt trials object, used to store intermediate results for all + optimization runs. Effective (and required) only when hyperopt + parameter searching is computed. + + sample_weight : multi types, default=None + Optional Numpy array of weights for the training samples, used + for weighting the loss function (during training only). Accepted + types are arrays or list/dict in case of multi-input/output. + + groups : array-like, default=None + Group labels for the samples used while splitting the dataset into + train/valid set. + + **fitargs : Additional fitting arguments, the same accepted in Keras + model.fit(...). + The validation set is automatically created accordingly to the + cv strategy. + + Returns + ------- + self : object + """ + + self._search(tuning_type='hyperopt', + x=x, y=y, + trials=trials, + sample_weight=sample_weight, + groups=groups, + **fitargs) + + return self \ No newline at end of file diff --git a/kerashypetune/utils.py b/kerashypetune/utils.py index 753fedf..9bd486b 100644 --- a/kerashypetune/utils.py +++ b/kerashypetune/utils.py @@ -13,10 +13,10 @@ def _check_param(values): if isinstance(values, (list, tuple, np.ndarray)): return list(set(values)) - - elif hasattr(values, 'rvs'): + elif 'scipy' in str(type(values)).lower(): + return values + elif 'hyperopt' in str(type(values)).lower(): return values - else: return [values] @@ -73,7 +73,7 @@ def _check_data(X, is_target=False): if len(set(data_len)) > 1: raise ValueError("Data must have the same cardinality. " - "Got {}".format(data_len)) + "Got {}.".format(data_len)) elif isinstance(X, dict): data_len = [] @@ -86,7 +86,7 @@ def _check_data(X, is_target=False): if len(set(data_len)) > 1: raise ValueError("Data must have the same cardinality. " - "Got {}".format(data_len)) + "Got {}.".format(data_len)) elif isinstance(X, np.ndarray): x = X @@ -117,12 +117,11 @@ def _is_multioutput(y): class ParameterSampler(object): - # modified from scikit-learn ParameterSampler """Generator on parameters sampled from given distributions. - Non-deterministic iterable over random candidate combinations for hyper- - parameter search. If all parameters are presented as a list, - sampling without replacement is performed. If at least one parameter - is given as a distribution, sampling with replacement is used. + If all parameters are presented as a list, sampling without replacement is + performed. If at least one parameter is given as a scipy distribution, + sampling with replacement is used. If all parameters are given as hyperopt + distributions Tree of Parzen Estimators searching from hyperopt is computed. It is highly recommended to use continuous distributions for continuous parameters. @@ -131,79 +130,98 @@ class ParameterSampler(object): param_distributions : dict Dictionary with parameters names (`str`) as keys and distributions or lists of parameters to try. Distributions must provide a ``rvs`` - method for sampling (such as those from scipy.stats.distributions). + method for random sampling (such as those from scipy.stats.distributions) + or be hyperopt distributions for bayesian searching. If a list is given, it is sampled uniformly. - If a list of dicts is given, first a dict is sampled uniformly, and - then a parameter is sampled using that dict as above. - n_iter : integer, default None - Number of parameter settings that are produced. + n_iter : integer, default=None + Number of parameter configurations that are produced. - random_state : int, default None + random_state : int, default=None Pass an int for reproducible output across multiple function calls. - is_random: bool, default True - If it's a random search. - Returns ------- - param_combi : list of tuple - list of sampled parameter combination + param_combi : list of dicts or dict of hyperopt distributions + Parameter combinations. + + searching_type : str + The searching algorithm used. """ - def __init__(self, param_distributions, n_iter=None, - random_state=None, is_random=False): + def __init__(self, param_distributions, n_iter=None, random_state=None): self.n_iter = n_iter self.random_state = random_state self.param_distributions = param_distributions - self.is_random = is_random def sample(self): + """Generator parameter combinations from given distributions.""" param_distributions = self.param_distributions.copy() - all_lists = all(not hasattr(p, "rvs") + is_grid = all(isinstance(p, list) + for p in param_distributions.values()) + is_random = all(isinstance(p, list) or 'scipy' in str(type(p)).lower() for p in param_distributions.values()) + is_hyperopt = all('hyperopt' in str(type(p)).lower() + or (len(p) < 2 if isinstance(p, list) else False) + for p in param_distributions.values()) - seed = (random.randint(1, 100) if self.random_state is None - else self.random_state + 1) - random.seed(seed) - - if all_lists: + if is_grid: param_combi = list(product(*param_distributions.values())) + param_combi = [ + dict(zip(param_distributions.keys(), combi)) + for combi in param_combi + ] + return param_combi, 'grid' - if self.is_random: - grid_size = len(param_combi) - if grid_size < self.n_iter: - raise ValueError( - "The total space of parameters {} is smaller " - "than n_iter={}. Try with KerasGridSearch.".format( - grid_size, self.n_iter)) - param_combi = random.sample(param_combi, self.n_iter) - - else: - + elif is_random: if self.n_iter is None: raise ValueError( - "n_iter must be an integer >0 when parameter " - "distributions are provided. Get None.") + "n_iter must be an integer >0 when scipy parameter " + "distributions are provided. Get None." + ) + + seed = (random.randint(1, 100) if self.random_state is None + else self.random_state + 1) + random.seed(seed) param_combi = [] k = self.n_iter for i in range(self.n_iter): dist = param_distributions.copy() - params = [] + combi = [] for j, v in enumerate(dist.values()): - if hasattr(v, "rvs"): - params.append(v.rvs(random_state=seed * (k + j))) + if 'scipy' in str(type(v)).lower(): + combi.append(v.rvs(random_state=seed * (k + j))) else: - params.append(v[random.randint(0, len(v) - 1)]) + combi.append(v[random.randint(0, len(v) - 1)]) k += i + j - param_combi.append(tuple(params)) + param_combi.append( + dict(zip(param_distributions.keys(), combi)) + ) + np.random.mtrand._rand - # reset seed - np.random.mtrand._rand + return param_combi, 'random' - return param_combi \ No newline at end of file + elif is_hyperopt: + if self.n_iter is None: + raise ValueError( + "n_iter must be an integer >0 when hyperopt " + "search spaces are provided. Get None." + ) + param_distributions = { + k: p[0] if isinstance(p, list) else p + for k, p in param_distributions.items() + } + + return param_distributions, 'hyperopt' + + else: + raise ValueError( + "Parameters not recognized. " + "Pass lists, scipy distributions (also in conjunction " + "with lists), or hyperopt search spaces." + ) \ No newline at end of file diff --git a/notebooks/Advance Usage.ipynb b/notebooks/Advance Usage.ipynb index b51d17c..b56cfaf 100644 --- a/notebooks/Advance Usage.ipynb +++ b/notebooks/Advance Usage.ipynb @@ -135,8 +135,8 @@ "***** (2/8) *****\n", "Search({'unit_1': 128, 'lr': 0.01, 'cond': True, 'epochs': 100, 'batch_size': 256})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00027: early stopping\n", - "SCORE: 0.9568 at epoch 18\n", + "Epoch 00023: early stopping\n", + "SCORE: 0.95444 at epoch 13\n", "\n", "***** (3/8) *****\n", "Search({'unit_1': 128, 'lr': 0.001, 'cond': False, 'epochs': 100, 'batch_size': 256})\n", @@ -147,8 +147,8 @@ "***** (4/8) *****\n", "Search({'unit_1': 128, 'lr': 0.001, 'cond': True, 'epochs': 100, 'batch_size': 256})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00034: early stopping\n", - "SCORE: 0.95039 at epoch 27\n", + "Epoch 00037: early stopping\n", + "SCORE: 0.95106 at epoch 27\n", "\n", "***** (5/8) *****\n", "Search({'unit_1': 64, 'lr': 0.01, 'cond': False, 'epochs': 100, 'batch_size': 256})\n", @@ -171,14 +171,14 @@ "***** (8/8) *****\n", "Search({'unit_1': 64, 'lr': 0.001, 'cond': True, 'epochs': 100, 'batch_size': 256})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00041: early stopping\n", - "SCORE: 0.94431 at epoch 31\n" + "Epoch 00042: early stopping\n", + "SCORE: 0.94465 at epoch 32\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 6, @@ -265,50 +265,50 @@ "***** (2/8) *****\n", "Search({'unit_1': 128, 'lr': 0.01, 'n_layer': 4, 'epochs': 100, 'batch_size': 256})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00024: early stopping\n", - "SCORE: 0.95478 at epoch 14\n", + "Epoch 00017: early stopping\n", + "SCORE: 0.94634 at epoch 10\n", "\n", "***** (3/8) *****\n", "Search({'unit_1': 128, 'lr': 0.001, 'n_layer': 3, 'epochs': 100, 'batch_size': 256})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00031: early stopping\n", - "SCORE: 0.94803 at epoch 30\n", + "Epoch 00040: early stopping\n", + "SCORE: 0.94904 at epoch 30\n", "\n", "***** (4/8) *****\n", "Search({'unit_1': 128, 'lr': 0.001, 'n_layer': 4, 'epochs': 100, 'batch_size': 256})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00036: early stopping\n", - "SCORE: 0.9514 at epoch 26\n", + "SCORE: 0.95039 at epoch 26\n", "\n", "***** (5/8) *****\n", "Search({'unit_1': 64, 'lr': 0.01, 'n_layer': 3, 'epochs': 100, 'batch_size': 256})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00022: early stopping\n", - "SCORE: 0.94296 at epoch 12\n", + "Epoch 00032: early stopping\n", + "SCORE: 0.95106 at epoch 22\n", "\n", "***** (6/8) *****\n", "Search({'unit_1': 64, 'lr': 0.01, 'n_layer': 4, 'epochs': 100, 'batch_size': 256})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00034: early stopping\n", - "SCORE: 0.94634 at epoch 24\n", + "Epoch 00028: early stopping\n", + "SCORE: 0.94465 at epoch 18\n", "\n", "***** (7/8) *****\n", "Search({'unit_1': 64, 'lr': 0.001, 'n_layer': 3, 'epochs': 100, 'batch_size': 256})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00041: early stopping\n", - "SCORE: 0.93858 at epoch 31\n", + "Epoch 00040: early stopping\n", + "SCORE: 0.93891 at epoch 31\n", "\n", "***** (8/8) *****\n", "Search({'unit_1': 64, 'lr': 0.001, 'n_layer': 4, 'epochs': 100, 'batch_size': 256})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00046: early stopping\n", - "SCORE: 0.9406 at epoch 36\n" + "Epoch 00037: early stopping\n", + "SCORE: 0.93959 at epoch 36\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 9, @@ -395,14 +395,14 @@ "***** (2/8) *****\n", "Search({'unit': 32, 'kernel': 3, 'lr': 0.1, 'layer_types': 'flat', 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00020: early stopping\n", - "SCORE: 0.92913 at epoch 18\n", + "Epoch 00012: early stopping\n", + "SCORE: 0.92676 at epoch 12\n", "\n", "***** (3/8) *****\n", "Search({'unit': 32, 'kernel': 3, 'lr': 0.01, 'layer_types': 'pool', 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00040: early stopping\n", - "SCORE: 0.71785 at epoch 35\n", + "SCORE: 0.71752 at epoch 35\n", "\n", "***** (4/8) *****\n", "Search({'unit': 32, 'kernel': 3, 'lr': 0.01, 'layer_types': 'flat', 'epochs': 100, 'batch_size': 512})\n", @@ -426,7 +426,7 @@ "Search({'unit': 64, 'kernel': 3, 'lr': 0.01, 'layer_types': 'pool', 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00054: early stopping\n", - "SCORE: 0.77354 at epoch 49\n", + "SCORE: 0.77489 at epoch 49\n", "\n", "***** (8/8) *****\n", "Search({'unit': 64, 'kernel': 3, 'lr': 0.01, 'layer_types': 'flat', 'epochs': 100, 'batch_size': 512})\n", @@ -438,7 +438,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 12, @@ -521,106 +521,106 @@ "16 trials detected for ('unit_1', 'unit_2', 'opt', 'lr', 'epochs', 'batch_size')\n", "\n", "***** (1/16) *****\n", - "Search({'unit_1': 128, 'unit_2': 64, 'opt': 'nadam', 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", - "Restoring model weights from the end of the best epoch.\n", - "Epoch 00012: early stopping\n", - "SCORE: 0.95073 at epoch 7\n", - "\n", - "***** (2/16) *****\n", - "Search({'unit_1': 128, 'unit_2': 64, 'opt': 'nadam', 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", - "Restoring model weights from the end of the best epoch.\n", - "Epoch 00030: early stopping\n", - "SCORE: 0.94938 at epoch 30\n", - "\n", - "***** (3/16) *****\n", "Search({'unit_1': 128, 'unit_2': 64, 'opt': 'adam', 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00016: early stopping\n", "SCORE: 0.95174 at epoch 13\n", "\n", - "***** (4/16) *****\n", + "***** (2/16) *****\n", "Search({'unit_1': 128, 'unit_2': 64, 'opt': 'adam', 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00033: early stopping\n", "SCORE: 0.95005 at epoch 28\n", "\n", - "***** (5/16) *****\n", - "Search({'unit_1': 128, 'unit_2': 32, 'opt': 'nadam', 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", + "***** (3/16) *****\n", + "Search({'unit_1': 128, 'unit_2': 64, 'opt': 'nadam', 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00015: early stopping\n", - "SCORE: 0.94668 at epoch 10\n", + "Epoch 00012: early stopping\n", + "SCORE: 0.95073 at epoch 7\n", "\n", - "***** (6/16) *****\n", - "Search({'unit_1': 128, 'unit_2': 32, 'opt': 'nadam', 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", + "***** (4/16) *****\n", + "Search({'unit_1': 128, 'unit_2': 64, 'opt': 'nadam', 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00027: early stopping\n", - "SCORE: 0.94398 at epoch 22\n", + "Epoch 00019: early stopping\n", + "SCORE: 0.94364 at epoch 19\n", "\n", - "***** (7/16) *****\n", + "***** (5/16) *****\n", "Search({'unit_1': 128, 'unit_2': 32, 'opt': 'adam', 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00019: early stopping\n", "SCORE: 0.9514 at epoch 14\n", "\n", - "***** (8/16) *****\n", + "***** (6/16) *****\n", "Search({'unit_1': 128, 'unit_2': 32, 'opt': 'adam', 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00029: early stopping\n", "SCORE: 0.94533 at epoch 24\n", "\n", - "***** (9/16) *****\n", - "Search({'unit_1': 64, 'unit_2': 64, 'opt': 'nadam', 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", + "***** (7/16) *****\n", + "Search({'unit_1': 128, 'unit_2': 32, 'opt': 'nadam', 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00019: early stopping\n", - "SCORE: 0.94701 at epoch 17\n", + "Epoch 00015: early stopping\n", + "SCORE: 0.94701 at epoch 10\n", "\n", - "***** (10/16) *****\n", - "Search({'unit_1': 64, 'unit_2': 64, 'opt': 'nadam', 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", + "***** (8/16) *****\n", + "Search({'unit_1': 128, 'unit_2': 32, 'opt': 'nadam', 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00035: early stopping\n", - "SCORE: 0.9406 at epoch 34\n", + "Epoch 00021: early stopping\n", + "SCORE: 0.93925 at epoch 18\n", "\n", - "***** (11/16) *****\n", + "***** (9/16) *****\n", "Search({'unit_1': 64, 'unit_2': 64, 'opt': 'adam', 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00019: early stopping\n", "SCORE: 0.95073 at epoch 14\n", "\n", - "***** (12/16) *****\n", + "***** (10/16) *****\n", "Search({'unit_1': 64, 'unit_2': 64, 'opt': 'adam', 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00029: early stopping\n", "SCORE: 0.93824 at epoch 28\n", "\n", - "***** (13/16) *****\n", - "Search({'unit_1': 64, 'unit_2': 32, 'opt': 'nadam', 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", + "***** (11/16) *****\n", + "Search({'unit_1': 64, 'unit_2': 64, 'opt': 'nadam', 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00017: early stopping\n", - "SCORE: 0.94735 at epoch 12\n", + "Epoch 00022: early stopping\n", + "SCORE: 0.94803 at epoch 22\n", "\n", - "***** (14/16) *****\n", - "Search({'unit_1': 64, 'unit_2': 32, 'opt': 'nadam', 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", + "***** (12/16) *****\n", + "Search({'unit_1': 64, 'unit_2': 64, 'opt': 'nadam', 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00027: early stopping\n", - "SCORE: 0.93486 at epoch 22\n", + "Epoch 00035: early stopping\n", + "SCORE: 0.9406 at epoch 34\n", "\n", - "***** (15/16) *****\n", + "***** (13/16) *****\n", "Search({'unit_1': 64, 'unit_2': 32, 'opt': 'adam', 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00023: early stopping\n", "SCORE: 0.9487 at epoch 18\n", "\n", - "***** (16/16) *****\n", + "***** (14/16) *****\n", "Search({'unit_1': 64, 'unit_2': 32, 'opt': 'adam', 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00042: early stopping\n", - "SCORE: 0.93925 at epoch 37\n" + "SCORE: 0.93925 at epoch 37\n", + "\n", + "***** (15/16) *****\n", + "Search({'unit_1': 64, 'unit_2': 32, 'opt': 'nadam', 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00017: early stopping\n", + "SCORE: 0.94735 at epoch 12\n", + "\n", + "***** (16/16) *****\n", + "Search({'unit_1': 64, 'unit_2': 32, 'opt': 'nadam', 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00027: early stopping\n", + "SCORE: 0.93486 at epoch 22\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 15, diff --git a/notebooks/Basic Usage BayesianSearch.ipynb b/notebooks/Basic Usage BayesianSearch.ipynb new file mode 100644 index 0000000..58f18b9 --- /dev/null +++ b/notebooks/Basic Usage BayesianSearch.ipynb @@ -0,0 +1,947 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import random\n", + "import numpy as np\n", + "\n", + "import tensorflow as tf\n", + "from tensorflow.keras.layers import *\n", + "from tensorflow.keras.models import *\n", + "from tensorflow.keras.callbacks import *\n", + "from tensorflow.keras.optimizers import Adam\n", + "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n", + "\n", + "from hyperopt import hp, Trials\n", + "from sklearn.model_selection import KFold\n", + "\n", + "from kerashypetune import KerasBayesianSearch, KerasBayesianSearchCV" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "def set_seed(seed):\n", + " \n", + " tf.random.set_seed(seed)\n", + " os.environ['PYTHONHASHSEED'] = str(seed)\n", + " np.random.seed(seed)\n", + " random.seed(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "((6036, 28, 28), (6036,), (2963, 28, 28), (2963,))" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()\n", + "\n", + "np.random.seed(33)\n", + "subset_train = np.random.uniform(0,1, len(y_train))\n", + "subset_test = np.random.uniform(0,1, len(y_test))\n", + "\n", + "x_train = x_train[subset_train < 0.1] / 255\n", + "y_train = y_train[subset_train < 0.1]\n", + "\n", + "x_test = x_test[subset_test < 0.3] / 255\n", + "y_test = y_test[subset_test < 0.3]\n", + "\n", + "x_train.shape, y_train.shape, x_test.shape, y_test.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def get_model(param):\n", + " \n", + " set_seed(33)\n", + " \n", + " model = Sequential()\n", + " model.add(Flatten())\n", + " model.add(Dense(param['unit_1'], activation='relu'))\n", + " model.add(Dense(param['unit_2'], activation='relu'))\n", + " model.add(Dense(10, activation='softmax'))\n", + " model.compile(optimizer=Adam(learning_rate=param['lr']), \n", + " loss='sparse_categorical_crossentropy',\n", + " metrics=['accuracy'])\n", + " \n", + " return model" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "param_grid = {\n", + " 'unit_1': 64 + hp.randint('unit_1', 64),\n", + " 'unit_2': 32 + hp.randint('unit_2', 96),\n", + " 'lr': hp.loguniform('lr', np.log(0.001), np.log(0.02)), \n", + " 'epochs': 100,\n", + " 'batch_size': 512\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Fixed validation search" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "15 trials detected for ('unit_1', 'unit_2', 'lr', 'epochs', 'batch_size')\n", + "\n", + "***** (1/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0016861379017239324, 'unit_1': 122, 'unit_2': 58})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00039: early stopping\n", + "SCORE: 0.9487 at epoch 29\n", + "\n", + "***** (2/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0014104783964893717, 'unit_1': 80, 'unit_2': 96})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00037: early stopping\n", + "SCORE: 0.9433 at epoch 37\n", + "\n", + "***** (3/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.006139711436597271, 'unit_1': 101, 'unit_2': 101})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00036: early stopping\n", + "SCORE: 0.95005 at epoch 26\n", + "\n", + "***** (4/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.018199685929694627, 'unit_1': 70, 'unit_2': 107})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00020: early stopping\n", + "SCORE: 0.94566 at epoch 13\n", + "\n", + "***** (5/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0030185075402515246, 'unit_1': 101, 'unit_2': 112})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00032: early stopping\n", + "SCORE: 0.95376 at epoch 22\n", + "\n", + "***** (6/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.001864571059195379, 'unit_1': 99, 'unit_2': 93})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00029: early stopping\n", + "SCORE: 0.94735 at epoch 28\n", + "\n", + "***** (7/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0028098615855990786, 'unit_1': 94, 'unit_2': 124})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00031: early stopping\n", + "SCORE: 0.94904 at epoch 21\n", + "\n", + "***** (8/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0029896673303287193, 'unit_1': 100, 'unit_2': 55})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00034: early stopping\n", + "SCORE: 0.95376 at epoch 24\n", + "\n", + "***** (9/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.004102183556799779, 'unit_1': 104, 'unit_2': 47})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00023: early stopping\n", + "SCORE: 0.94398 at epoch 13\n", + "\n", + "***** (10/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.002569664537891012, 'unit_1': 94, 'unit_2': 103})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00045: early stopping\n", + "SCORE: 0.95106 at epoch 35\n", + "\n", + "***** (11/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.007617003049628987, 'unit_1': 94, 'unit_2': 120})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00023: early stopping\n", + "SCORE: 0.9514 at epoch 13\n", + "\n", + "***** (12/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.003369536974829801, 'unit_1': 75, 'unit_2': 114})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00029: early stopping\n", + "SCORE: 0.9487 at epoch 21\n", + "\n", + "***** (13/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.002097940683587956, 'unit_1': 123, 'unit_2': 61})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00031: early stopping\n", + "SCORE: 0.95343 at epoch 21\n", + "\n", + "***** (14/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.001488081463762352, 'unit_1': 84, 'unit_2': 126})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00039: early stopping\n", + "SCORE: 0.946 at epoch 36\n", + "\n", + "***** (15/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0032272609715520537, 'unit_1': 94, 'unit_2': 120})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00035: early stopping\n", + "SCORE: 0.95039 at epoch 30\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "es = EarlyStopping(patience=10, verbose=1, min_delta=0.001, monitor='val_accuracy', mode='auto', restore_best_weights=True)\n", + "\n", + "hypermodel = get_model \n", + "# to pass external arguments to get_model use lambda function\n", + "# ==> hypermodel = lambda x: get_model(param=x, ...)\n", + "# callable(hypermodel) ==> True\n", + "\n", + "kbs = KerasBayesianSearch(hypermodel, param_grid, n_iter=15, sampling_seed=33,\n", + " monitor='val_accuracy', greater_is_better=True, tuner_verbose=1)\n", + "kbs.search(x_train, y_train, trials=Trials(), validation_data=(x_test, y_test), callbacks=[es])" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[0.9487,\n", + " 0.9433,\n", + " 0.95005,\n", + " 0.94566,\n", + " 0.95376,\n", + " 0.94735,\n", + " 0.94904,\n", + " 0.95376,\n", + " 0.94398,\n", + " 0.95106,\n", + " 0.9514,\n", + " 0.9487,\n", + " 0.95343,\n", + " 0.946,\n", + " 0.95039]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kbs.scores" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.95376" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kbs.best_score" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'batch_size': 512,\n", + " 'epochs': 22,\n", + " 'lr': 0.0030185075402515246,\n", + " 'unit_1': 101,\n", + " 'unit_2': 112,\n", + " 'steps_per_epoch': 12}" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kbs.best_params" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kbs.best_model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Fixed validation search Generator" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "gen = ImageDataGenerator(\n", + " rotation_range=90,\n", + " width_shift_range=0.1,\n", + " height_shift_range=0.1,\n", + " zoom_range=0.2)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "15 trials detected for ('unit_1', 'unit_2', 'lr', 'epochs', 'batch_size')\n", + "\n", + "***** (1/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0016861379017239324, 'unit_1': 122, 'unit_2': 58})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00029: early stopping\n", + "SCORE: 0.8839 at epoch 24\n", + "\n", + "***** (2/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0014104783964893717, 'unit_1': 80, 'unit_2': 96})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00023: early stopping\n", + "SCORE: 0.84914 at epoch 18\n", + "\n", + "***** (3/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.006139711436597271, 'unit_1': 101, 'unit_2': 101})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00018: early stopping\n", + "SCORE: 0.86196 at epoch 13\n", + "\n", + "***** (4/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.018199685929694627, 'unit_1': 70, 'unit_2': 107})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00027: early stopping\n", + "SCORE: 0.82923 at epoch 22\n", + "\n", + "***** (5/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0030185075402515246, 'unit_1': 101, 'unit_2': 112})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00037: early stopping\n", + "SCORE: 0.89976 at epoch 32\n", + "\n", + "***** (6/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.001864571059195379, 'unit_1': 99, 'unit_2': 93})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00027: early stopping\n", + "SCORE: 0.88424 at epoch 22\n", + "\n", + "***** (7/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0028098615855990786, 'unit_1': 94, 'unit_2': 124})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00028: early stopping\n", + "SCORE: 0.87951 at epoch 23\n", + "\n", + "***** (8/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0029896673303287193, 'unit_1': 100, 'unit_2': 55})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00026: early stopping\n", + "SCORE: 0.87715 at epoch 21\n", + "\n", + "***** (9/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.004102183556799779, 'unit_1': 104, 'unit_2': 47})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00030: early stopping\n", + "SCORE: 0.8758 at epoch 25\n", + "\n", + "***** (10/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.002569664537891012, 'unit_1': 94, 'unit_2': 103})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00035: early stopping\n", + "SCORE: 0.88896 at epoch 30\n", + "\n", + "***** (11/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.007617003049628987, 'unit_1': 94, 'unit_2': 120})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00033: early stopping\n", + "SCORE: 0.8866 at epoch 28\n", + "\n", + "***** (12/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.003369536974829801, 'unit_1': 75, 'unit_2': 114})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00030: early stopping\n", + "SCORE: 0.8731 at epoch 28\n", + "\n", + "***** (13/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.002097940683587956, 'unit_1': 123, 'unit_2': 61})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00024: early stopping\n", + "SCORE: 0.87006 at epoch 19\n", + "\n", + "***** (14/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.001488081463762352, 'unit_1': 84, 'unit_2': 126})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00040: early stopping\n", + "SCORE: 0.87546 at epoch 35\n", + "\n", + "***** (15/15) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0032272609715520537, 'unit_1': 94, 'unit_2': 120})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00041: early stopping\n", + "SCORE: 0.89571 at epoch 36\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "es = EarlyStopping(patience=5, verbose=1, min_delta=0.001, monitor='val_accuracy', mode='auto', restore_best_weights=True)\n", + "\n", + "hypermodel = get_model\n", + "# to pass external arguments to get_model use lambda function\n", + "# ==> hypermodel = lambda x: get_model(param=x, ...)\n", + "# callable(hypermodel) ==> True\n", + "\n", + "kbs = KerasBayesianSearch(hypermodel, param_grid, n_iter=15, sampling_seed=33,\n", + " monitor='val_accuracy', greater_is_better=True, tuner_verbose=1)\n", + "kbs.search(gen.flow(np.expand_dims(x_train,-1), y_train, batch_size=param_grid['batch_size'], seed=33), \n", + " trials=Trials(),\n", + " validation_data=(np.expand_dims(x_test,-1), y_test), \n", + " callbacks=[es], steps_per_epoch=len(x_train)//param_grid['batch_size'])" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[0.8839,\n", + " 0.84914,\n", + " 0.86196,\n", + " 0.82923,\n", + " 0.89976,\n", + " 0.88424,\n", + " 0.87951,\n", + " 0.87715,\n", + " 0.8758,\n", + " 0.88896,\n", + " 0.8866,\n", + " 0.8731,\n", + " 0.87006,\n", + " 0.87546,\n", + " 0.89571]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kbs.scores" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.89976" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kbs.best_score" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'batch_size': 512,\n", + " 'epochs': 32,\n", + " 'lr': 0.0030185075402515246,\n", + " 'unit_1': 101,\n", + " 'unit_2': 112,\n", + " 'steps_per_epoch': 11}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kbs.best_params" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kbs.best_model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Cross validation search" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "((8999, 28, 28), (8999,))" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X = np.concatenate([x_train, x_test])\n", + "y = np.concatenate([y_train, y_test])\n", + "\n", + "X.shape, y.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "##################\n", + "### Fold 001 ###\n", + "##################\n", + "\n", + "5 trials detected for ('unit_1', 'unit_2', 'lr', 'epochs', 'batch_size')\n", + "\n", + "***** (1/5) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0016861379017239324, 'unit_1': 122, 'unit_2': 58})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00028: early stopping\n", + "SCORE: 0.942 at epoch 28\n", + "\n", + "***** (2/5) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0014104783964893717, 'unit_1': 80, 'unit_2': 96})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00030: early stopping\n", + "SCORE: 0.93633 at epoch 30\n", + "\n", + "***** (3/5) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.006139711436597271, 'unit_1': 101, 'unit_2': 101})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00024: early stopping\n", + "SCORE: 0.94633 at epoch 23\n", + "\n", + "***** (4/5) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.018199685929694627, 'unit_1': 70, 'unit_2': 107})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00024: early stopping\n", + "SCORE: 0.946 at epoch 23\n", + "\n", + "***** (5/5) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0030185075402515246, 'unit_1': 101, 'unit_2': 112})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00033: early stopping\n", + "SCORE: 0.94267 at epoch 27\n", + "\n", + "##################\n", + "### Fold 002 ###\n", + "##################\n", + "\n", + "5 trials detected for ('unit_1', 'unit_2', 'lr', 'epochs', 'batch_size')\n", + "\n", + "***** (1/5) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0016861379017239324, 'unit_1': 122, 'unit_2': 58})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00042: early stopping\n", + "SCORE: 0.951 at epoch 42\n", + "\n", + "***** (2/5) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0014104783964893717, 'unit_1': 80, 'unit_2': 96})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00034: early stopping\n", + "SCORE: 0.94567 at epoch 26\n", + "\n", + "***** (3/5) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.006139711436597271, 'unit_1': 101, 'unit_2': 101})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00022: early stopping\n", + "SCORE: 0.95233 at epoch 14\n", + "\n", + "***** (4/5) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.018199685929694627, 'unit_1': 70, 'unit_2': 107})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00015: early stopping\n", + "SCORE: 0.946 at epoch 14\n", + "\n", + "***** (5/5) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0030185075402515246, 'unit_1': 101, 'unit_2': 112})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00029: early stopping\n", + "SCORE: 0.95133 at epoch 19\n", + "\n", + "##################\n", + "### Fold 003 ###\n", + "##################\n", + "\n", + "5 trials detected for ('unit_1', 'unit_2', 'lr', 'epochs', 'batch_size')\n", + "\n", + "***** (1/5) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0016861379017239324, 'unit_1': 122, 'unit_2': 58})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00027: early stopping\n", + "SCORE: 0.94498 at epoch 21\n", + "\n", + "***** (2/5) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0014104783964893717, 'unit_1': 80, 'unit_2': 96})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00029: early stopping\n", + "SCORE: 0.94598 at epoch 21\n", + "\n", + "***** (3/5) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.006139711436597271, 'unit_1': 101, 'unit_2': 101})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00026: early stopping\n", + "SCORE: 0.95265 at epoch 16\n", + "\n", + "***** (4/5) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.018199685929694627, 'unit_1': 70, 'unit_2': 107})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00022: early stopping\n", + "SCORE: 0.94765 at epoch 12\n", + "\n", + "***** (5/5) *****\n", + "Search({'batch_size': 512, 'epochs': 100, 'lr': 0.0030185075402515246, 'unit_1': 101, 'unit_2': 112})\n", + "Restoring model weights from the end of the best epoch.\n", + "Epoch 00039: early stopping\n", + "SCORE: 0.95198 at epoch 29\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cv = KFold(n_splits=3, random_state=33, shuffle=True)\n", + "\n", + "es = EarlyStopping(patience=10, verbose=1, min_delta=0.001, monitor='val_accuracy', mode='auto', restore_best_weights=True)\n", + "\n", + "hypermodel = get_model\n", + "# to pass external arguments to get_model use lambda function\n", + "# ==> hypermodel = lambda x: get_model(param=x, ...)\n", + "\n", + "kbs = KerasBayesianSearchCV(hypermodel, param_grid, cv=cv, n_iter=5, sampling_seed=33,\n", + " monitor='val_accuracy', greater_is_better=True, tuner_verbose=1)\n", + "kbs.search(X, y, trials=Trials(), callbacks=[es])" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'fold 1': [0.942, 0.93633, 0.94633, 0.946, 0.94267],\n", + " 'fold 2': [0.951, 0.94567, 0.95233, 0.946, 0.95133],\n", + " 'fold 3': [0.94498, 0.94598, 0.95265, 0.94765, 0.95198]}" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kbs.folds_scores" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'fold 1': 0.94633, 'fold 2': 0.95233, 'fold 3': 0.95265}" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kbs.folds_best_score" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'fold 1': {'batch_size': 512,\n", + " 'epochs': 23,\n", + " 'lr': 0.006139711436597271,\n", + " 'unit_1': 101,\n", + " 'unit_2': 101,\n", + " 'steps_per_epoch': 12},\n", + " 'fold 2': {'batch_size': 512,\n", + " 'epochs': 14,\n", + " 'lr': 0.006139711436597271,\n", + " 'unit_1': 101,\n", + " 'unit_2': 101,\n", + " 'steps_per_epoch': 12},\n", + " 'fold 3': {'batch_size': 512,\n", + " 'epochs': 16,\n", + " 'lr': 0.006139711436597271,\n", + " 'unit_1': 101,\n", + " 'unit_2': 101,\n", + " 'steps_per_epoch': 12}}" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kbs.folds_best_params" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'fold 1': ,\n", + " 'fold 2': ,\n", + " 'fold 3': }" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kbs.folds_best_models" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.95044" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kbs.best_params_score" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'batch_size': 512,\n", + " 'epochs': 23,\n", + " 'lr': 0.006139711436597271,\n", + " 'unit_1': 101,\n", + " 'unit_2': 101,\n", + " 'steps_per_epoch': 12},\n", + " {'batch_size': 512,\n", + " 'epochs': 14,\n", + " 'lr': 0.006139711436597271,\n", + " 'unit_1': 101,\n", + " 'unit_2': 101,\n", + " 'steps_per_epoch': 12},\n", + " {'batch_size': 512,\n", + " 'epochs': 16,\n", + " 'lr': 0.006139711436597271,\n", + " 'unit_1': 101,\n", + " 'unit_2': 101,\n", + " 'steps_per_epoch': 12}]" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "kbs.best_params" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/Basic Usage GridSearch.ipynb b/notebooks/Basic Usage GridSearch.ipynb index 96e9c43..07c9077 100644 --- a/notebooks/Basic Usage GridSearch.ipynb +++ b/notebooks/Basic Usage GridSearch.ipynb @@ -176,7 +176,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 6, @@ -269,7 +269,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 10, @@ -316,56 +316,56 @@ "***** (1/8) *****\n", "Search({'unit_1': 128, 'unit_2': 64, 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00029: early stopping\n", - "SCORE: 0.89436 at epoch 24\n", + "Epoch 00020: early stopping\n", + "SCORE: 0.85353 at epoch 15\n", "\n", "***** (2/8) *****\n", "Search({'unit_1': 128, 'unit_2': 64, 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00039: early stopping\n", - "SCORE: 0.87884 at epoch 39\n", + "Epoch 00031: early stopping\n", + "SCORE: 0.86905 at epoch 26\n", "\n", "***** (3/8) *****\n", "Search({'unit_1': 128, 'unit_2': 32, 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00046: early stopping\n", - "SCORE: 0.90179 at epoch 41\n", + "Epoch 00049: early stopping\n", + "SCORE: 0.90348 at epoch 44\n", "\n", "***** (4/8) *****\n", "Search({'unit_1': 128, 'unit_2': 32, 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00031: early stopping\n", - "SCORE: 0.86196 at epoch 26\n", + "Epoch 00035: early stopping\n", + "SCORE: 0.86365 at epoch 34\n", "\n", "***** (5/8) *****\n", "Search({'unit_1': 64, 'unit_2': 64, 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00030: early stopping\n", - "SCORE: 0.8596 at epoch 25\n", + "Epoch 00035: early stopping\n", + "SCORE: 0.86466 at epoch 30\n", "\n", "***** (6/8) *****\n", "Search({'unit_1': 64, 'unit_2': 64, 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00034: early stopping\n", - "SCORE: 0.83496 at epoch 29\n", + "Epoch 00037: early stopping\n", + "SCORE: 0.82889 at epoch 32\n", "\n", "***** (7/8) *****\n", "Search({'unit_1': 64, 'unit_2': 32, 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00025: early stopping\n", - "SCORE: 0.83733 at epoch 20\n", + "Epoch 00028: early stopping\n", + "SCORE: 0.83733 at epoch 28\n", "\n", "***** (8/8) *****\n", "Search({'unit_1': 64, 'unit_2': 32, 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00048: early stopping\n", - "SCORE: 0.85218 at epoch 43\n" + "Epoch 00050: early stopping\n", + "SCORE: 0.8353 at epoch 45\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 12, @@ -395,7 +395,7 @@ { "data": { "text/plain": [ - "[0.89436, 0.87884, 0.90179, 0.86196, 0.8596, 0.83496, 0.83733, 0.85218]" + "[0.85353, 0.86905, 0.90348, 0.86365, 0.86466, 0.82889, 0.83733, 0.8353]" ] }, "execution_count": 13, @@ -415,7 +415,7 @@ { "data": { "text/plain": [ - "0.90179" + "0.90348" ] }, "execution_count": 14, @@ -438,7 +438,7 @@ "{'unit_1': 128,\n", " 'unit_2': 32,\n", " 'lr': 0.01,\n", - " 'epochs': 41,\n", + " 'epochs': 44,\n", " 'batch_size': 512,\n", " 'steps_per_epoch': 11}" ] @@ -460,7 +460,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 16, @@ -551,8 +551,8 @@ "***** (6/8) *****\n", "Search({'unit_1': 64, 'unit_2': 64, 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00038: early stopping\n", - "SCORE: 0.93333 at epoch 29\n", + "Epoch 00042: early stopping\n", + "SCORE: 0.93433 at epoch 32\n", "\n", "***** (7/8) *****\n", "Search({'unit_1': 64, 'unit_2': 32, 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", @@ -575,14 +575,14 @@ "***** (1/8) *****\n", "Search({'unit_1': 128, 'unit_2': 64, 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00030: early stopping\n", - "SCORE: 0.95767 at epoch 20\n", + "Epoch 00029: early stopping\n", + "SCORE: 0.958 at epoch 23\n", "\n", "***** (2/8) *****\n", "Search({'unit_1': 128, 'unit_2': 64, 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00036: early stopping\n", - "SCORE: 0.95233 at epoch 36\n", + "SCORE: 0.95267 at epoch 36\n", "\n", "***** (3/8) *****\n", "Search({'unit_1': 128, 'unit_2': 32, 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", @@ -606,13 +606,13 @@ "Search({'unit_1': 64, 'unit_2': 64, 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00057: early stopping\n", - "SCORE: 0.94 at epoch 47\n", + "SCORE: 0.93967 at epoch 47\n", "\n", "***** (7/8) *****\n", "Search({'unit_1': 64, 'unit_2': 32, 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00033: early stopping\n", - "SCORE: 0.948 at epoch 30\n", + "Epoch 00025: early stopping\n", + "SCORE: 0.94733 at epoch 15\n", "\n", "***** (8/8) *****\n", "Search({'unit_1': 64, 'unit_2': 32, 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", @@ -654,7 +654,7 @@ "Search({'unit_1': 64, 'unit_2': 64, 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00022: early stopping\n", - "SCORE: 0.94632 at epoch 12\n", + "SCORE: 0.94765 at epoch 17\n", "\n", "***** (6/8) *****\n", "Search({'unit_1': 64, 'unit_2': 64, 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", @@ -671,14 +671,14 @@ "***** (8/8) *****\n", "Search({'unit_1': 64, 'unit_2': 32, 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00041: early stopping\n", - "SCORE: 0.93965 at epoch 36\n" + "Epoch 00058: early stopping\n", + "SCORE: 0.94365 at epoch 48\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 18, @@ -709,16 +709,16 @@ { "data": { "text/plain": [ - "{'fold 1': [0.94767, 0.944, 0.94567, 0.935, 0.94667, 0.93333, 0.944, 0.93467],\n", - " 'fold 2': [0.95767, 0.95233, 0.95633, 0.94733, 0.95233, 0.94, 0.948, 0.94],\n", + "{'fold 1': [0.94767, 0.944, 0.94567, 0.935, 0.94667, 0.93433, 0.944, 0.93467],\n", + " 'fold 2': [0.958, 0.95267, 0.95633, 0.94733, 0.95233, 0.93967, 0.94733, 0.94],\n", " 'fold 3': [0.95365,\n", " 0.94698,\n", " 0.95065,\n", " 0.94532,\n", - " 0.94632,\n", + " 0.94765,\n", " 0.94265,\n", " 0.94898,\n", - " 0.93965]}" + " 0.94365]}" ] }, "execution_count": 19, @@ -738,7 +738,7 @@ { "data": { "text/plain": [ - "{'fold 1': 0.94767, 'fold 2': 0.95767, 'fold 3': 0.95365}" + "{'fold 1': 0.94767, 'fold 2': 0.958, 'fold 3': 0.95365}" ] }, "execution_count": 20, @@ -767,7 +767,7 @@ " 'fold 2': {'unit_1': 128,\n", " 'unit_2': 64,\n", " 'lr': 0.01,\n", - " 'epochs': 20,\n", + " 'epochs': 23,\n", " 'batch_size': 512,\n", " 'steps_per_epoch': 12},\n", " 'fold 3': {'unit_1': 128,\n", @@ -795,9 +795,9 @@ { "data": { "text/plain": [ - "{'fold 1': ,\n", - " 'fold 2': ,\n", - " 'fold 3': }" + "{'fold 1': ,\n", + " 'fold 2': ,\n", + " 'fold 3': }" ] }, "execution_count": 22, @@ -817,7 +817,7 @@ { "data": { "text/plain": [ - "0.953" + "0.95311" ] }, "execution_count": 23, @@ -846,7 +846,7 @@ " {'unit_1': 128,\n", " 'unit_2': 64,\n", " 'lr': 0.01,\n", - " 'epochs': 20,\n", + " 'epochs': 23,\n", " 'batch_size': 512,\n", " 'steps_per_epoch': 12},\n", " {'unit_1': 128,\n", diff --git a/notebooks/Basic Usage RandomSearch.ipynb b/notebooks/Basic Usage RandomSearch.ipynb index 1e4f45d..a02076c 100644 --- a/notebooks/Basic Usage RandomSearch.ipynb +++ b/notebooks/Basic Usage RandomSearch.ipynb @@ -134,8 +134,8 @@ "***** (2/15) *****\n", "Search({'unit_1': 128, 'unit_2': 105, 'lr': 0.018917299504794916, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00041: early stopping\n", - "SCORE: 0.94836 at epoch 31\n", + "Epoch 00030: early stopping\n", + "SCORE: 0.94668 at epoch 21\n", "\n", "***** (3/15) *****\n", "Search({'unit_1': 128, 'unit_2': 81, 'lr': 0.006387142993161844, 'epochs': 100, 'batch_size': 512})\n", @@ -170,14 +170,14 @@ "***** (8/15) *****\n", "Search({'unit_1': 128, 'unit_2': 72, 'lr': 0.028446264284627223, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00022: early stopping\n", - "SCORE: 0.93453 at epoch 12\n", + "Epoch 00029: early stopping\n", + "SCORE: 0.93621 at epoch 19\n", "\n", "***** (9/15) *****\n", "Search({'unit_1': 64, 'unit_2': 79, 'lr': 0.0002597559131018589, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00085: early stopping\n", - "SCORE: 0.93284 at epoch 84\n", + "Epoch 00078: early stopping\n", + "SCORE: 0.93216 at epoch 75\n", "\n", "***** (10/15) *****\n", "Search({'unit_1': 64, 'unit_2': 89, 'lr': 0.027450030060679365, 'epochs': 100, 'batch_size': 512})\n", @@ -200,8 +200,8 @@ "***** (13/15) *****\n", "Search({'unit_1': 128, 'unit_2': 89, 'lr': 0.04442377859898533, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00018: early stopping\n", - "SCORE: 0.91731 at epoch 8\n", + "Epoch 00033: early stopping\n", + "SCORE: 0.92035 at epoch 23\n", "\n", "***** (14/15) *****\n", "Search({'unit_1': 64, 'unit_2': 69, 'lr': 0.0026994807935171675, 'epochs': 100, 'batch_size': 512})\n", @@ -212,14 +212,14 @@ "***** (15/15) *****\n", "Search({'unit_1': 128, 'unit_2': 32, 'lr': 0.07961407194405414, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00026: early stopping\n", - "SCORE: 0.90786 at epoch 16\n" + "Epoch 00022: early stopping\n", + "SCORE: 0.9109 at epoch 12\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 6, @@ -235,9 +235,9 @@ "# ==> hypermodel = lambda x: get_model(param=x, ...)\n", "# callable(hypermodel) ==> True\n", "\n", - "kgs = KerasRandomSearch(hypermodel, param_grid, n_iter=15, sampling_seed=33,\n", + "krs = KerasRandomSearch(hypermodel, param_grid, n_iter=15, sampling_seed=33,\n", " monitor='val_accuracy', greater_is_better=True, tuner_verbose=1)\n", - "kgs.search(x_train, y_train, validation_data=(x_test, y_test), callbacks=[es])" + "krs.search(x_train, y_train, validation_data=(x_test, y_test), callbacks=[es])" ] }, { @@ -249,20 +249,20 @@ "data": { "text/plain": [ "[0.93453,\n", - " 0.94836,\n", + " 0.94668,\n", " 0.95714,\n", " 0.93149,\n", " 0.94465,\n", " 0.95174,\n", " 0.92575,\n", - " 0.93453,\n", - " 0.93284,\n", + " 0.93621,\n", + " 0.93216,\n", " 0.9406,\n", " 0.95174,\n", " 0.93858,\n", - " 0.91731,\n", + " 0.92035,\n", " 0.94296,\n", - " 0.90786]" + " 0.9109]" ] }, "execution_count": 7, @@ -271,7 +271,7 @@ } ], "source": [ - "kgs.scores" + "krs.scores" ] }, { @@ -291,7 +291,7 @@ } ], "source": [ - "kgs.best_score" + "krs.best_score" ] }, { @@ -316,7 +316,7 @@ } ], "source": [ - "kgs.best_params" + "krs.best_params" ] }, { @@ -327,7 +327,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 10, @@ -336,7 +336,7 @@ } ], "source": [ - "kgs.best_model" + "krs.best_model" ] }, { @@ -374,98 +374,98 @@ "***** (1/15) *****\n", "Search({'unit_1': 64, 'unit_2': 61, 'lr': 0.00026941073027491154, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00086: early stopping\n", - "SCORE: 0.82551 at epoch 82\n", + "Epoch 00085: early stopping\n", + "SCORE: 0.82686 at epoch 80\n", "\n", "***** (2/15) *****\n", "Search({'unit_1': 128, 'unit_2': 105, 'lr': 0.018917299504794916, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00039: early stopping\n", - "SCORE: 0.86061 at epoch 34\n", + "Epoch 00028: early stopping\n", + "SCORE: 0.85656 at epoch 23\n", "\n", "***** (3/15) *****\n", "Search({'unit_1': 128, 'unit_2': 81, 'lr': 0.006387142993161844, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00019: early stopping\n", - "SCORE: 0.86736 at epoch 14\n", + "Epoch 00023: early stopping\n", + "SCORE: 0.89673 at epoch 18\n", "\n", "***** (4/15) *****\n", "Search({'unit_1': 64, 'unit_2': 96, 'lr': 0.00016980298333942208, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00092: early stopping\n", - "SCORE: 0.80628 at epoch 87\n", + "Epoch 00057: early stopping\n", + "SCORE: 0.75397 at epoch 52\n", "\n", "***** (5/15) *****\n", "Search({'unit_1': 128, 'unit_2': 125, 'lr': 0.00031100502312585046, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00051: early stopping\n", - "SCORE: 0.85116 at epoch 46\n", + "Epoch 00063: early stopping\n", + "SCORE: 0.86365 at epoch 58\n", "\n", "***** (6/15) *****\n", "Search({'unit_1': 128, 'unit_2': 38, 'lr': 0.008604539745472692, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00014: early stopping\n", - "SCORE: 0.83631 at epoch 9\n", + "Epoch 00037: early stopping\n", + "SCORE: 0.89875 at epoch 32\n", "\n", "***** (7/15) *****\n", "Search({'unit_1': 128, 'unit_2': 67, 'lr': 0.048643553143575324, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00017: early stopping\n", - "SCORE: 0.54877 at epoch 12\n", + "Epoch 00033: early stopping\n", + "SCORE: 0.67972 at epoch 28\n", "\n", "***** (8/15) *****\n", "Search({'unit_1': 128, 'unit_2': 72, 'lr': 0.028446264284627223, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00021: early stopping\n", - "SCORE: 0.75937 at epoch 16\n", + "Epoch 00027: early stopping\n", + "SCORE: 0.78299 at epoch 22\n", "\n", "***** (9/15) *****\n", "Search({'unit_1': 64, 'unit_2': 79, 'lr': 0.0002597559131018589, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00065: early stopping\n", - "SCORE: 0.79447 at epoch 63\n", + "Epoch 00083: early stopping\n", + "SCORE: 0.82146 at epoch 78\n", "\n", "***** (10/15) *****\n", "Search({'unit_1': 64, 'unit_2': 89, 'lr': 0.027450030060679365, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00021: early stopping\n", - "SCORE: 0.77827 at epoch 16\n", + "Epoch 00023: early stopping\n", + "SCORE: 0.74789 at epoch 18\n", "\n", "***** (11/15) *****\n", "Search({'unit_1': 64, 'unit_2': 77, 'lr': 0.011180289095021183, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00032: early stopping\n", - "SCORE: 0.85218 at epoch 27\n", + "Epoch 00036: early stopping\n", + "SCORE: 0.86703 at epoch 31\n", "\n", "***** (12/15) *****\n", "Search({'unit_1': 64, 'unit_2': 111, 'lr': 0.000289005574642862, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00092: early stopping\n", - "SCORE: 0.84846 at epoch 87\n", + "Epoch 00064: early stopping\n", + "SCORE: 0.82484 at epoch 62\n", "\n", "***** (13/15) *****\n", "Search({'unit_1': 128, 'unit_2': 89, 'lr': 0.04442377859898533, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00017: early stopping\n", - "SCORE: 0.61323 at epoch 12\n", + "SCORE: 0.54877 at epoch 12\n", "\n", "***** (14/15) *****\n", "Search({'unit_1': 64, 'unit_2': 69, 'lr': 0.0026994807935171675, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00019: early stopping\n", - "SCORE: 0.8245 at epoch 14\n", + "Epoch 00021: early stopping\n", + "SCORE: 0.83901 at epoch 17\n", "\n", "***** (15/15) *****\n", "Search({'unit_1': 128, 'unit_2': 32, 'lr': 0.07961407194405414, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00016: early stopping\n", - "SCORE: 0.31792 at epoch 11\n" + "Epoch 00006: early stopping\n", + "SCORE: 0.1134 at epoch 1\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 12, @@ -481,9 +481,9 @@ "# ==> hypermodel = lambda x: get_model(param=x, ...)\n", "# callable(hypermodel) ==> True\n", "\n", - "kgs = KerasRandomSearch(hypermodel, param_grid, n_iter=15, sampling_seed=33,\n", + "krs = KerasRandomSearch(hypermodel, param_grid, n_iter=15, sampling_seed=33,\n", " monitor='val_accuracy', greater_is_better=True, tuner_verbose=1)\n", - "kgs.search(gen.flow(np.expand_dims(x_train,-1), y_train, batch_size=param_grid['batch_size'], seed=33), \n", + "krs.search(gen.flow(np.expand_dims(x_train,-1), y_train, batch_size=param_grid['batch_size'], seed=33), \n", " validation_data=(np.expand_dims(x_test,-1), y_test), \n", " callbacks=[es], steps_per_epoch=len(x_train)//param_grid['batch_size'])" ] @@ -496,21 +496,21 @@ { "data": { "text/plain": [ - "[0.82551,\n", - " 0.86061,\n", - " 0.86736,\n", - " 0.80628,\n", - " 0.85116,\n", - " 0.83631,\n", + "[0.82686,\n", + " 0.85656,\n", + " 0.89673,\n", + " 0.75397,\n", + " 0.86365,\n", + " 0.89875,\n", + " 0.67972,\n", + " 0.78299,\n", + " 0.82146,\n", + " 0.74789,\n", + " 0.86703,\n", + " 0.82484,\n", " 0.54877,\n", - " 0.75937,\n", - " 0.79447,\n", - " 0.77827,\n", - " 0.85218,\n", - " 0.84846,\n", - " 0.61323,\n", - " 0.8245,\n", - " 0.31792]" + " 0.83901,\n", + " 0.1134]" ] }, "execution_count": 13, @@ -519,7 +519,7 @@ } ], "source": [ - "kgs.scores" + "krs.scores" ] }, { @@ -530,7 +530,7 @@ { "data": { "text/plain": [ - "0.86736" + "0.89875" ] }, "execution_count": 14, @@ -539,7 +539,7 @@ } ], "source": [ - "kgs.best_score" + "krs.best_score" ] }, { @@ -551,9 +551,9 @@ "data": { "text/plain": [ "{'unit_1': 128,\n", - " 'unit_2': 81,\n", - " 'lr': 0.006387142993161844,\n", - " 'epochs': 14,\n", + " 'unit_2': 38,\n", + " 'lr': 0.008604539745472692,\n", + " 'epochs': 32,\n", " 'batch_size': 512,\n", " 'steps_per_epoch': 11}" ] @@ -564,7 +564,7 @@ } ], "source": [ - "kgs.best_params" + "krs.best_params" ] }, { @@ -575,7 +575,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 16, @@ -584,7 +584,7 @@ } ], "source": [ - "kgs.best_model" + "krs.best_model" ] }, { @@ -636,14 +636,14 @@ "***** (1/5) *****\n", "Search({'unit_1': 64, 'unit_2': 97, 'lr': 0.0005535560552210636, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00042: early stopping\n", - "SCORE: 0.92667 at epoch 36\n", + "Epoch 00064: early stopping\n", + "SCORE: 0.93033 at epoch 54\n", "\n", "***** (2/5) *****\n", "Search({'unit_1': 128, 'unit_2': 77, 'lr': 0.0025150330161023593, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00035: early stopping\n", - "SCORE: 0.942 at epoch 25\n", + "SCORE: 0.94233 at epoch 27\n", "\n", "***** (3/5) *****\n", "Search({'unit_1': 128, 'unit_2': 37, 'lr': 0.0005324197618194066, 'epochs': 100, 'batch_size': 512})\n", @@ -690,14 +690,14 @@ "***** (4/5) *****\n", "Search({'unit_1': 64, 'unit_2': 114, 'lr': 0.0005304760422961851, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00058: early stopping\n", - "SCORE: 0.93867 at epoch 57\n", + "Epoch 00044: early stopping\n", + "SCORE: 0.93533 at epoch 38\n", "\n", "***** (5/5) *****\n", "Search({'unit_1': 128, 'unit_2': 125, 'lr': 0.010310596407937588, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00022: early stopping\n", - "SCORE: 0.95533 at epoch 12\n", + "Epoch 00020: early stopping\n", + "SCORE: 0.955 at epoch 11\n", "\n", "##################\n", "### Fold 003 ###\n", @@ -720,8 +720,8 @@ "***** (3/5) *****\n", "Search({'unit_1': 128, 'unit_2': 37, 'lr': 0.0005324197618194066, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00049: early stopping\n", - "SCORE: 0.94165 at epoch 46\n", + "Epoch 00044: early stopping\n", + "SCORE: 0.94098 at epoch 39\n", "\n", "***** (4/5) *****\n", "Search({'unit_1': 64, 'unit_2': 114, 'lr': 0.0005304760422961851, 'epochs': 100, 'batch_size': 512})\n", @@ -739,7 +739,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 18, @@ -756,9 +756,9 @@ "# to pass external arguments to get_model use lambda function\n", "# ==> hypermodel = lambda x: get_model(param=x, ...)\n", "\n", - "kgs = KerasRandomSearchCV(hypermodel, param_grid, cv=cv, n_iter=5, sampling_seed=33,\n", + "krs = KerasRandomSearchCV(hypermodel, param_grid, cv=cv, n_iter=5, sampling_seed=33,\n", " monitor='val_accuracy', greater_is_better=True, tuner_verbose=1)\n", - "kgs.search(X, y, callbacks=[es])" + "krs.search(X, y, callbacks=[es])" ] }, { @@ -771,9 +771,9 @@ { "data": { "text/plain": [ - "{'fold 1': [0.92667, 0.942, 0.93733, 0.93533, 0.951],\n", - " 'fold 2': [0.93767, 0.95367, 0.94333, 0.93867, 0.95533],\n", - " 'fold 3': [0.93765, 0.95365, 0.94165, 0.94532, 0.95499]}" + "{'fold 1': [0.93033, 0.94233, 0.93733, 0.93533, 0.951],\n", + " 'fold 2': [0.93767, 0.95367, 0.94333, 0.93533, 0.955],\n", + " 'fold 3': [0.93765, 0.95365, 0.94098, 0.94532, 0.95499]}" ] }, "execution_count": 19, @@ -782,7 +782,7 @@ } ], "source": [ - "kgs.folds_scores" + "krs.folds_scores" ] }, { @@ -793,7 +793,7 @@ { "data": { "text/plain": [ - "{'fold 1': 0.951, 'fold 2': 0.95533, 'fold 3': 0.95499}" + "{'fold 1': 0.951, 'fold 2': 0.955, 'fold 3': 0.95499}" ] }, "execution_count": 20, @@ -802,7 +802,7 @@ } ], "source": [ - "kgs.folds_best_score" + "krs.folds_best_score" ] }, { @@ -822,7 +822,7 @@ " 'fold 2': {'unit_1': 128,\n", " 'unit_2': 125,\n", " 'lr': 0.010310596407937588,\n", - " 'epochs': 12,\n", + " 'epochs': 11,\n", " 'batch_size': 512,\n", " 'steps_per_epoch': 12},\n", " 'fold 3': {'unit_1': 128,\n", @@ -839,7 +839,7 @@ } ], "source": [ - "kgs.folds_best_params" + "krs.folds_best_params" ] }, { @@ -850,9 +850,9 @@ { "data": { "text/plain": [ - "{'fold 1': ,\n", - " 'fold 2': ,\n", - " 'fold 3': }" + "{'fold 1': ,\n", + " 'fold 2': ,\n", + " 'fold 3': }" ] }, "execution_count": 22, @@ -861,7 +861,7 @@ } ], "source": [ - "kgs.folds_best_models" + "krs.folds_best_models" ] }, { @@ -872,7 +872,7 @@ { "data": { "text/plain": [ - "0.95377" + "0.95366" ] }, "execution_count": 23, @@ -881,7 +881,7 @@ } ], "source": [ - "kgs.best_params_score" + "krs.best_params_score" ] }, { @@ -901,7 +901,7 @@ " {'unit_1': 128,\n", " 'unit_2': 125,\n", " 'lr': 0.010310596407937588,\n", - " 'epochs': 12,\n", + " 'epochs': 11,\n", " 'batch_size': 512,\n", " 'steps_per_epoch': 12},\n", " {'unit_1': 128,\n", @@ -918,7 +918,7 @@ } ], "source": [ - "kgs.best_params" + "krs.best_params" ] } ], diff --git a/notebooks/Multi-Input Multi-Output Search.ipynb b/notebooks/Multi-Input Multi-Output Search.ipynb index 4641deb..8fd1032 100644 --- a/notebooks/Multi-Input Multi-Output Search.ipynb +++ b/notebooks/Multi-Input Multi-Output Search.ipynb @@ -139,13 +139,13 @@ "Search({'unit_1': 128, 'unit_2': 64, 'unit_hid': 32, 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00012: early stopping\n", - "SCORE: 0.22427 at epoch 7\n", + "SCORE: 0.22122 at epoch 7\n", "\n", "***** (2/16) *****\n", "Search({'unit_1': 128, 'unit_2': 64, 'unit_hid': 32, 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00026: early stopping\n", - "SCORE: 0.2332 at epoch 24\n", + "Epoch 00029: early stopping\n", + "SCORE: 0.23336 at epoch 24\n", "\n", "***** (3/16) *****\n", "Search({'unit_1': 128, 'unit_2': 64, 'unit_hid': 16, 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", @@ -157,7 +157,7 @@ "Search({'unit_1': 128, 'unit_2': 64, 'unit_hid': 16, 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00029: early stopping\n", - "SCORE: 0.23569 at epoch 29\n", + "SCORE: 0.23555 at epoch 29\n", "\n", "***** (5/16) *****\n", "Search({'unit_1': 128, 'unit_2': 32, 'unit_hid': 32, 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", @@ -169,7 +169,7 @@ "Search({'unit_1': 128, 'unit_2': 32, 'unit_hid': 32, 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00029: early stopping\n", - "SCORE: 0.23084 at epoch 24\n", + "SCORE: 0.23076 at epoch 24\n", "\n", "***** (7/16) *****\n", "Search({'unit_1': 128, 'unit_2': 32, 'unit_hid': 16, 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", @@ -181,7 +181,7 @@ "Search({'unit_1': 128, 'unit_2': 32, 'unit_hid': 16, 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00029: early stopping\n", - "SCORE: 0.2333 at epoch 28\n", + "SCORE: 0.23331 at epoch 26\n", "\n", "***** (9/16) *****\n", "Search({'unit_1': 64, 'unit_2': 64, 'unit_hid': 32, 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", @@ -193,7 +193,7 @@ "Search({'unit_1': 64, 'unit_2': 64, 'unit_hid': 32, 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00029: early stopping\n", - "SCORE: 0.23422 at epoch 24\n", + "SCORE: 0.23405 at epoch 24\n", "\n", "***** (11/16) *****\n", "Search({'unit_1': 64, 'unit_2': 64, 'unit_hid': 16, 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", @@ -205,25 +205,25 @@ "Search({'unit_1': 64, 'unit_2': 64, 'unit_hid': 16, 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00029: early stopping\n", - "SCORE: 0.25652 at epoch 24\n", + "SCORE: 0.25631 at epoch 29\n", "\n", "***** (13/16) *****\n", "Search({'unit_1': 64, 'unit_2': 32, 'unit_hid': 32, 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00015: early stopping\n", - "SCORE: 0.24808 at epoch 10\n", + "SCORE: 0.25209 at epoch 10\n", "\n", "***** (14/16) *****\n", "Search({'unit_1': 64, 'unit_2': 32, 'unit_hid': 32, 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00034: early stopping\n", - "SCORE: 0.24891 at epoch 29\n", + "SCORE: 0.2489 at epoch 29\n", "\n", "***** (15/16) *****\n", "Search({'unit_1': 64, 'unit_2': 32, 'unit_hid': 16, 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00015: early stopping\n", - "SCORE: 0.2828 at epoch 10\n", + "SCORE: 0.28165 at epoch 10\n", "\n", "***** (16/16) *****\n", "Search({'unit_1': 64, 'unit_2': 32, 'unit_hid': 16, 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", @@ -235,7 +235,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 6, @@ -261,7 +261,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "[0.22427, 0.2332, 0.26374, 0.23569, 0.25271, 0.23084, 0.29183, 0.2333, 0.24856, 0.23422, 0.26177, 0.25652, 0.24808, 0.24891, 0.2828, 0.26808]\n" + "[0.22122, 0.23336, 0.26374, 0.23555, 0.25271, 0.23076, 0.29183, 0.23331, 0.24856, 0.23405, 0.26177, 0.25631, 0.25209, 0.2489, 0.28165, 0.26808]\n" ] } ], @@ -277,7 +277,7 @@ { "data": { "text/plain": [ - "0.22427" + "0.22122" ] }, "execution_count": 8, @@ -323,7 +323,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 10, @@ -479,7 +479,7 @@ "Search({'unit_1': 128, 'unit_2': 64, 'activ': 'relu', 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00025: early stopping\n", - "SCORE: 0.17568 at epoch 20\n", + "SCORE: 0.17547 at epoch 20\n", "\n", "***** (5/32) *****\n", "Search({'unit_1': 128, 'unit_2': 64, 'activ': 'elu', 'lr': 0.01, 'epochs': 100, 'batch_size': 256})\n", @@ -496,8 +496,8 @@ "***** (7/32) *****\n", "Search({'unit_1': 128, 'unit_2': 64, 'activ': 'elu', 'lr': 0.001, 'epochs': 100, 'batch_size': 256})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00014: early stopping\n", - "SCORE: 0.18686 at epoch 9\n", + "Epoch 00013: early stopping\n", + "SCORE: 0.18757 at epoch 9\n", "\n", "***** (8/32) *****\n", "Search({'unit_1': 128, 'unit_2': 64, 'activ': 'elu', 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", @@ -515,7 +515,7 @@ "Search({'unit_1': 128, 'unit_2': 32, 'activ': 'relu', 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00021: early stopping\n", - "SCORE: 0.18344 at epoch 16\n", + "SCORE: 0.18007 at epoch 16\n", "\n", "***** (11/32) *****\n", "Search({'unit_1': 128, 'unit_2': 32, 'activ': 'relu', 'lr': 0.001, 'epochs': 100, 'batch_size': 256})\n", @@ -527,7 +527,7 @@ "Search({'unit_1': 128, 'unit_2': 32, 'activ': 'relu', 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00027: early stopping\n", - "SCORE: 0.17574 at epoch 22\n", + "SCORE: 0.1759 at epoch 22\n", "\n", "***** (13/32) *****\n", "Search({'unit_1': 128, 'unit_2': 32, 'activ': 'elu', 'lr': 0.01, 'epochs': 100, 'batch_size': 256})\n", @@ -569,13 +569,13 @@ "Search({'unit_1': 64, 'unit_2': 64, 'activ': 'relu', 'lr': 0.001, 'epochs': 100, 'batch_size': 256})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00018: early stopping\n", - "SCORE: 0.18343 at epoch 13\n", + "SCORE: 0.1839 at epoch 13\n", "\n", "***** (20/32) *****\n", "Search({'unit_1': 64, 'unit_2': 64, 'activ': 'relu', 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00025: early stopping\n", - "SCORE: 0.1851 at epoch 22\n", + "SCORE: 0.18528 at epoch 22\n", "\n", "***** (21/32) *****\n", "Search({'unit_1': 64, 'unit_2': 64, 'activ': 'elu', 'lr': 0.01, 'epochs': 100, 'batch_size': 256})\n", @@ -599,7 +599,7 @@ "Search({'unit_1': 64, 'unit_2': 64, 'activ': 'elu', 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00019: early stopping\n", - "SCORE: 0.19545 at epoch 15\n", + "SCORE: 0.19547 at epoch 15\n", "\n", "***** (25/32) *****\n", "Search({'unit_1': 64, 'unit_2': 32, 'activ': 'relu', 'lr': 0.01, 'epochs': 100, 'batch_size': 256})\n", @@ -616,26 +616,26 @@ "***** (27/32) *****\n", "Search({'unit_1': 64, 'unit_2': 32, 'activ': 'relu', 'lr': 0.001, 'epochs': 100, 'batch_size': 256})\n", "Restoring model weights from the end of the best epoch.\n", - "Epoch 00018: early stopping\n", - "SCORE: 0.19666 at epoch 13\n", + "Epoch 00017: early stopping\n", + "SCORE: 0.19793 at epoch 13\n", "\n", "***** (28/32) *****\n", "Search({'unit_1': 64, 'unit_2': 32, 'activ': 'relu', 'lr': 0.001, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00027: early stopping\n", - "SCORE: 0.19022 at epoch 22\n", + "SCORE: 0.18984 at epoch 22\n", "\n", "***** (29/32) *****\n", "Search({'unit_1': 64, 'unit_2': 32, 'activ': 'elu', 'lr': 0.01, 'epochs': 100, 'batch_size': 256})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00008: early stopping\n", - "SCORE: 0.17806 at epoch 3\n", + "SCORE: 0.17752 at epoch 3\n", "\n", "***** (30/32) *****\n", "Search({'unit_1': 64, 'unit_2': 32, 'activ': 'elu', 'lr': 0.01, 'epochs': 100, 'batch_size': 512})\n", "Restoring model weights from the end of the best epoch.\n", "Epoch 00010: early stopping\n", - "SCORE: 0.1804 at epoch 5\n", + "SCORE: 0.18043 at epoch 5\n", "\n", "***** (31/32) *****\n", "Search({'unit_1': 64, 'unit_2': 32, 'activ': 'elu', 'lr': 0.001, 'epochs': 100, 'batch_size': 256})\n", @@ -653,7 +653,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 15, @@ -679,7 +679,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "[0.18378, 0.18168, 0.16768, 0.17568, 0.1787, 0.17906, 0.18686, 0.19279, 0.18653, 0.18344, 0.18323, 0.17574, 0.17126, 0.17232, 0.19072, 0.20002, 0.18135, 0.17467, 0.18343, 0.1851, 0.1742, 0.1817, 0.19459, 0.19545, 0.1916, 0.19075, 0.19666, 0.19022, 0.17806, 0.1804, 0.20165, 0.20478]\n" + "[0.18378, 0.18168, 0.16768, 0.17547, 0.1787, 0.17906, 0.18757, 0.19279, 0.18653, 0.18007, 0.18323, 0.1759, 0.17126, 0.17232, 0.19072, 0.20002, 0.18135, 0.17467, 0.1839, 0.18528, 0.1742, 0.1817, 0.19459, 0.19547, 0.1916, 0.19075, 0.19793, 0.18984, 0.17752, 0.18043, 0.20165, 0.20478]\n" ] } ], @@ -741,7 +741,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 19, diff --git a/setup.py b/setup.py index 659770a..941e1e7 100644 --- a/setup.py +++ b/setup.py @@ -3,19 +3,20 @@ HERE = pathlib.Path(__file__).parent -VERSION = '0.1.3' +VERSION = '0.2.0' PACKAGE_NAME = 'keras-hypetune' AUTHOR = 'Marco Cerliani' AUTHOR_EMAIL = 'cerlymarco@gmail.com' URL = 'https://github.com/cerlymarco/keras-hypetune' LICENSE = 'MIT' -DESCRIPTION = 'A friendly python package for Keras Hyperparameters Tuning based only on NumPy.' +DESCRIPTION = 'A friendly python package for Keras Hyperparameters Tuning based only on NumPy and Hyperopt.' LONG_DESCRIPTION = (HERE / "README.md").read_text() LONG_DESC_TYPE = "text/markdown" INSTALL_REQUIRES = [ - 'numpy' + 'numpy', + 'hyperopt==0.2.5' ] setup(name=PACKAGE_NAME,