From c81eeef18836fd3b25a987cb744f9afadb47c30c Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Thu, 16 Jul 2020 18:47:23 -0700 Subject: [PATCH 01/47] warm start --- examples/warm_start.py | 37 +++++++++++++++++++++++++++++++++ tune_sklearn/_trainable.py | 14 +++++++++++-- tune_sklearn/tune_basesearch.py | 2 +- 3 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 examples/warm_start.py diff --git a/examples/warm_start.py b/examples/warm_start.py new file mode 100644 index 00000000..b14da2aa --- /dev/null +++ b/examples/warm_start.py @@ -0,0 +1,37 @@ +""" +An example training an SGDClassifier, performing grid search +using TuneGridSearchCV. + +This example uses early stopping to further improve runtimes +by eliminating worse hyperparameter choices early based off +of its average test score from cross validation. +""" + +from tune_sklearn import TuneGridSearchCV +from sklearn.linear_model import PassiveAggressiveClassifier +from sklearn import datasets +from sklearn.model_selection import train_test_split +from ray.tune.schedulers import MedianStoppingRule +import numpy as np + +digits = datasets.load_digits() +x = digits.data +y = digits.target +x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=.2) + +clf = PassiveAggressiveClassifier() +parameter_grid = {"C": [1e-4, 1e-1, 1, 2]} + +scheduler = MedianStoppingRule(grace_period=10.0) + +tune_search = TuneGridSearchCV( + clf, + parameter_grid, + early_stopping=scheduler, + max_iters=10, +) +tune_search.fit(x_train, y_train) + +pred = tune_search.predict(x_test) +accuracy = np.count_nonzero(np.array(pred) == np.array(y_test)) / len(pred) +print(accuracy) diff --git a/tune_sklearn/_trainable.py b/tune_sklearn/_trainable.py index 2424e3a3..7ac9aff0 100644 --- a/tune_sklearn/_trainable.py +++ b/tune_sklearn/_trainable.py @@ -11,6 +11,8 @@ from pickle import PicklingError import ray.cloudpickle as cpickle import warnings +from warnings import simplefilter +from sklearn.exceptions import ConvergenceWarning class _Trainable(Trainable): @@ -51,6 +53,9 @@ def _setup(self, config): n_splits = self.cv.get_n_splits(self.X, self.y) self.fold_scores = np.zeros(n_splits) self.fold_train_scores = np.zeros(n_splits) + if not hasattr(self.estimator, "partial_fit"): + self.estimator_config["warm_start"] = True + self.estimator_config["max_iter"] = 1 for i in range(n_splits): self.estimator[i].set_params(**self.estimator_config) else: @@ -86,8 +91,13 @@ def _train(self): self.y, test, train_indices=train) - self.estimator[i].partial_fit(X_train, y_train, - np.unique(self.y)) + if hasattr(self.estimator, "partial_fit"): + self.estimator[i].partial_fit(X_train, y_train, + np.unique(self.y)) + else: + simplefilter("ignore", category=ConvergenceWarning) + self.estimator[i].fit(X_train, y_train) + if self.return_train_score: self.fold_train_scores[i] = self.scoring( self.estimator[i], X_train, y_train) diff --git a/tune_sklearn/tune_basesearch.py b/tune_sklearn/tune_basesearch.py index 88e89061..d9f53216 100644 --- a/tune_sklearn/tune_basesearch.py +++ b/tune_sklearn/tune_basesearch.py @@ -405,7 +405,7 @@ def _can_early_stop(self): """ return (hasattr(self.estimator, "partial_fit") - and callable(getattr(self.estimator, "partial_fit", None))) + and callable(getattr(self.estimator, "partial_fit", None))) or (hasattr(self.estimator, "warm_start") and hasattr(self.estimator, "max_iter")) def _fill_config_hyperparam(self, config): """Fill in the ``config`` dictionary with the hyperparameters. From a70d40f8c115fb462d19832584d41cc5cf0974a0 Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Thu, 16 Jul 2020 20:32:25 -0700 Subject: [PATCH 02/47] lint --- setup.py | 2 +- tune_sklearn/tune_basesearch.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index ea0a8cf5..54a59bb2 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -from setuptools import setup +from distutils.core import setup setup( name="tune_sklearn", diff --git a/tune_sklearn/tune_basesearch.py b/tune_sklearn/tune_basesearch.py index d9f53216..67268603 100644 --- a/tune_sklearn/tune_basesearch.py +++ b/tune_sklearn/tune_basesearch.py @@ -404,8 +404,10 @@ def _can_early_stop(self): bool: if the estimator can early stop """ - return (hasattr(self.estimator, "partial_fit") - and callable(getattr(self.estimator, "partial_fit", None))) or (hasattr(self.estimator, "warm_start") and hasattr(self.estimator, "max_iter")) + return (hasattr(self.estimator, "partial_fit") and callable( + getattr(self.estimator, "partial_fit", + None))) or (hasattr(self.estimator, "warm_start") + and hasattr(self.estimator, "max_iter")) def _fill_config_hyperparam(self, config): """Fill in the ``config`` dictionary with the hyperparameters. From 04b8fec0bd7060afefaaf603c3ecc99e2280986e Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Thu, 16 Jul 2020 20:39:21 -0700 Subject: [PATCH 03/47] lint and fix example --- examples/warm_start.py | 10 ++++------ setup.py | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/examples/warm_start.py b/examples/warm_start.py index b14da2aa..655eb48b 100644 --- a/examples/warm_start.py +++ b/examples/warm_start.py @@ -8,18 +8,16 @@ """ from tune_sklearn import TuneGridSearchCV -from sklearn.linear_model import PassiveAggressiveClassifier -from sklearn import datasets from sklearn.model_selection import train_test_split +from sklearn.datasets import load_iris +from sklearn.linear_model import LogisticRegression from ray.tune.schedulers import MedianStoppingRule import numpy as np -digits = datasets.load_digits() -x = digits.data -y = digits.target +x, y = load_iris(return_X_y=True) x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=.2) -clf = PassiveAggressiveClassifier() +clf = LogisticRegression() parameter_grid = {"C": [1e-4, 1e-1, 1, 2]} scheduler = MedianStoppingRule(grace_period=10.0) diff --git a/setup.py b/setup.py index 54a59bb2..ea0a8cf5 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -from distutils.core import setup +from setuptools import setup setup( name="tune_sklearn", From 3022ad816a8fda89fac9593752986349e9ac211a Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Thu, 16 Jul 2020 20:53:55 -0700 Subject: [PATCH 04/47] exclude DTs and ensembles --- tune_sklearn/tune_basesearch.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tune_sklearn/tune_basesearch.py b/tune_sklearn/tune_basesearch.py index 67268603..62e846ba 100644 --- a/tune_sklearn/tune_basesearch.py +++ b/tune_sklearn/tune_basesearch.py @@ -404,10 +404,14 @@ def _can_early_stop(self): bool: if the estimator can early stop """ + sklearn_module = ".".join( + getattr(self.estimator, "__module__", None).split(".")[:-1]) return (hasattr(self.estimator, "partial_fit") and callable( getattr(self.estimator, "partial_fit", None))) or (hasattr(self.estimator, "warm_start") - and hasattr(self.estimator, "max_iter")) + and hasattr(self.estimator, "max_iter") + and sklearn_module != "sklearn.tree" + and sklearn_module != "sklearn.ensemble") def _fill_config_hyperparam(self, config): """Fill in the ``config`` dictionary with the hyperparameters. From a23806cb3edbad230c41a28a1fc0e9c2a0a152b7 Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Thu, 30 Jul 2020 17:48:59 -0700 Subject: [PATCH 05/47] lgbm early stop --- tune_sklearn/_trainable.py | 13 +++++++++++-- tune_sklearn/tune_basesearch.py | 6 ++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/tune_sklearn/_trainable.py b/tune_sklearn/_trainable.py index 850e51bc..7f056ebc 100644 --- a/tune_sklearn/_trainable.py +++ b/tune_sklearn/_trainable.py @@ -6,6 +6,7 @@ from sklearn.base import clone from sklearn.model_selection import cross_validate from sklearn.utils.metaestimators import _safe_split +from lightgbm import LGBMModel import numpy as np import os from pickle import PicklingError @@ -54,6 +55,10 @@ def _setup(self, config): self.fold_train_scores = np.zeros(n_splits) for i in range(n_splits): self.estimator[i].set_params(**self.estimator_config) + + self.is_lgbm = isinstance(self.estimator[0], LGBMModel) + if self.is_lgbm: + self.saved_models = [None for _ in range(n_splits)] else: self.estimator.set_params(**self.estimator_config) @@ -87,8 +92,12 @@ def _train(self): self.y, test, train_indices=train) - self.estimator[i].partial_fit(X_train, y_train, - np.unique(self.y)) + if self.is_lgbm: + self.saved_models[i] = self.estimator[i].fit( + X_train, y_train, self.saved_models[i]) + else: + self.estimator[i].partial_fit(X_train, y_train, + np.unique(self.y)) if self.return_train_score: self.fold_train_scores[i] = self.scoring( self.estimator[i], X_train, y_train) diff --git a/tune_sklearn/tune_basesearch.py b/tune_sklearn/tune_basesearch.py index 8ccc75e6..5db8a5e0 100644 --- a/tune_sklearn/tune_basesearch.py +++ b/tune_sklearn/tune_basesearch.py @@ -19,6 +19,7 @@ from sklearn.base import is_classifier from sklearn.base import clone from sklearn.exceptions import NotFittedError +from lightgbm import LGBMModel import ray from ray.tune.schedulers import ( PopulationBasedTraining, AsyncHyperBandScheduler, HyperBandScheduler, @@ -422,8 +423,9 @@ def _can_early_stop(self): bool: if the estimator can early stop """ - return (hasattr(self.estimator, "partial_fit") - and callable(getattr(self.estimator, "partial_fit", None))) + return (hasattr(self.estimator, "partial_fit") and callable( + getattr(self.estimator, "partial_fit", None))) or (isinstance( + self.estimator, LGBMModel)) def _fill_config_hyperparam(self, config): """Fill in the ``config`` dictionary with the hyperparameters. From 74890a5f11bb43cd0d4c4b7b7164a7a62f4923a1 Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Thu, 30 Jul 2020 22:53:49 -0700 Subject: [PATCH 06/47] example --- examples/lgbm.py | 2 +- tune_sklearn/_trainable.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/lgbm.py b/examples/lgbm.py index d4a2d9cd..4af8e6e0 100644 --- a/examples/lgbm.py +++ b/examples/lgbm.py @@ -29,7 +29,7 @@ "subsample_freq": [20] } -gs = TuneSearchCV(model, param_dists, n_iter=5, scoring="accuracy") +gs = TuneSearchCV(model, param_dists, n_iter=5, scoring="accuracy", early_stopping="MedianStoppingRule") gs.fit(X_train, y_train) print(gs.cv_results_) diff --git a/tune_sklearn/_trainable.py b/tune_sklearn/_trainable.py index 7f056ebc..d0a105c8 100644 --- a/tune_sklearn/_trainable.py +++ b/tune_sklearn/_trainable.py @@ -94,7 +94,7 @@ def _train(self): train_indices=train) if self.is_lgbm: self.saved_models[i] = self.estimator[i].fit( - X_train, y_train, self.saved_models[i]) + X_train, y_train, init_model=self.saved_models[i]) else: self.estimator[i].partial_fit(X_train, y_train, np.unique(self.y)) From 87aa7b9260349504fc849b3a86db8003488ffd76 Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Fri, 31 Jul 2020 10:19:35 -0700 Subject: [PATCH 07/47] xgb early stop --- examples/xgbclassifier.py | 1 + tune_sklearn/_trainable.py | 7 ++++++- tune_sklearn/tune_basesearch.py | 6 ++++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/examples/xgbclassifier.py b/examples/xgbclassifier.py index 8b31f5e3..4b9d2fe8 100644 --- a/examples/xgbclassifier.py +++ b/examples/xgbclassifier.py @@ -37,6 +37,7 @@ xgb, param_distributions=params, n_iter=3, + early_stopping="MedianStoppingRule", # use_gpu=True # Commented out for testing on travis, # but this is how you would use gpu ) diff --git a/tune_sklearn/_trainable.py b/tune_sklearn/_trainable.py index d0a105c8..c806124e 100644 --- a/tune_sklearn/_trainable.py +++ b/tune_sklearn/_trainable.py @@ -7,6 +7,7 @@ from sklearn.model_selection import cross_validate from sklearn.utils.metaestimators import _safe_split from lightgbm import LGBMModel +from xgboost.sklearn import XGBModel import numpy as np import os from pickle import PicklingError @@ -57,7 +58,8 @@ def _setup(self, config): self.estimator[i].set_params(**self.estimator_config) self.is_lgbm = isinstance(self.estimator[0], LGBMModel) - if self.is_lgbm: + self.is_xgb = isinstance(self.estimator[0], XGBModel) + if self.is_lgbm or self.is_xgb: self.saved_models = [None for _ in range(n_splits)] else: self.estimator.set_params(**self.estimator_config) @@ -95,6 +97,9 @@ def _train(self): if self.is_lgbm: self.saved_models[i] = self.estimator[i].fit( X_train, y_train, init_model=self.saved_models[i]) + elif self.is_xgb: + self.saved_models[i] = self.estimator[i].fit( + X_train, y_train, xgb_model=self.saved_models[i]) else: self.estimator[i].partial_fit(X_train, y_train, np.unique(self.y)) diff --git a/tune_sklearn/tune_basesearch.py b/tune_sklearn/tune_basesearch.py index 5db8a5e0..1113af33 100644 --- a/tune_sklearn/tune_basesearch.py +++ b/tune_sklearn/tune_basesearch.py @@ -20,6 +20,7 @@ from sklearn.base import clone from sklearn.exceptions import NotFittedError from lightgbm import LGBMModel +from xgboost.sklearn import XGBModel import ray from ray.tune.schedulers import ( PopulationBasedTraining, AsyncHyperBandScheduler, HyperBandScheduler, @@ -424,8 +425,9 @@ def _can_early_stop(self): """ return (hasattr(self.estimator, "partial_fit") and callable( - getattr(self.estimator, "partial_fit", None))) or (isinstance( - self.estimator, LGBMModel)) + getattr(self.estimator, "partial_fit", None))) or isinstance( + self.estimator, LGBMModel) + or isinstance(self.estimator, XGBModel) def _fill_config_hyperparam(self, config): """Fill in the ``config`` dictionary with the hyperparameters. From 06fd20a66b2e0f295c5ef2fa331f59aff289d5d3 Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Fri, 31 Jul 2020 10:20:40 -0700 Subject: [PATCH 08/47] lint --- examples/lgbm.py | 7 ++++++- tune_sklearn/tune_basesearch.py | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/examples/lgbm.py b/examples/lgbm.py index 4af8e6e0..e1a4e8d5 100644 --- a/examples/lgbm.py +++ b/examples/lgbm.py @@ -29,7 +29,12 @@ "subsample_freq": [20] } -gs = TuneSearchCV(model, param_dists, n_iter=5, scoring="accuracy", early_stopping="MedianStoppingRule") +gs = TuneSearchCV( + model, + param_dists, + n_iter=5, + scoring="accuracy", + early_stopping="MedianStoppingRule") gs.fit(X_train, y_train) print(gs.cv_results_) diff --git a/tune_sklearn/tune_basesearch.py b/tune_sklearn/tune_basesearch.py index 1113af33..f7752582 100644 --- a/tune_sklearn/tune_basesearch.py +++ b/tune_sklearn/tune_basesearch.py @@ -426,8 +426,8 @@ def _can_early_stop(self): """ return (hasattr(self.estimator, "partial_fit") and callable( getattr(self.estimator, "partial_fit", None))) or isinstance( - self.estimator, LGBMModel) - or isinstance(self.estimator, XGBModel) + self.estimator, LGBMModel) or isinstance( + self.estimator, XGBModel) def _fill_config_hyperparam(self, config): """Fill in the ``config`` dictionary with the hyperparameters. From 8b610e94dc28e9d4005d53f362b6f6d212a5a57b Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Fri, 31 Jul 2020 12:41:18 -0700 Subject: [PATCH 09/47] fix xgb --- tune_sklearn/_trainable.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tune_sklearn/_trainable.py b/tune_sklearn/_trainable.py index c806124e..f24c220d 100644 --- a/tune_sklearn/_trainable.py +++ b/tune_sklearn/_trainable.py @@ -98,8 +98,9 @@ def _train(self): self.saved_models[i] = self.estimator[i].fit( X_train, y_train, init_model=self.saved_models[i]) elif self.is_xgb: - self.saved_models[i] = self.estimator[i].fit( + self.estimator[i].fit( X_train, y_train, xgb_model=self.saved_models[i]) + self.saved_models[i] = self.estimator[i].get_booster() else: self.estimator[i].partial_fit(X_train, y_train, np.unique(self.y)) From 54c885748e84e01c580ec8bb8b30c9488e487bdb Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Fri, 31 Jul 2020 21:28:09 -0700 Subject: [PATCH 10/47] no early stop xgb --- examples/xgbclassifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/xgbclassifier.py b/examples/xgbclassifier.py index 4b9d2fe8..89c64a00 100644 --- a/examples/xgbclassifier.py +++ b/examples/xgbclassifier.py @@ -37,7 +37,7 @@ xgb, param_distributions=params, n_iter=3, - early_stopping="MedianStoppingRule", + # early_stopping="MedianStoppingRule", # use_gpu=True # Commented out for testing on travis, # but this is how you would use gpu ) From 313330a2bb14611044d899d200cf30bbf00a523b Mon Sep 17 00:00:00 2001 From: Anthony Yu <13611707+anthonyhsyu@users.noreply.github.com> Date: Sat, 1 Aug 2020 13:54:34 -0700 Subject: [PATCH 11/47] Quick doc update for example --- examples/warm_start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/warm_start.py b/examples/warm_start.py index 655eb48b..e4831ab0 100644 --- a/examples/warm_start.py +++ b/examples/warm_start.py @@ -1,5 +1,5 @@ """ -An example training an SGDClassifier, performing grid search +An example training a LogisticRegression model, performing grid search using TuneGridSearchCV. This example uses early stopping to further improve runtimes From c0adf2d74abfdd0e322389d0c4b52792cc0b9647 Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Thu, 6 Aug 2020 18:23:02 -0700 Subject: [PATCH 12/47] xgb early stop example --- examples/xgbclassifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/xgbclassifier.py b/examples/xgbclassifier.py index 89c64a00..4b9d2fe8 100644 --- a/examples/xgbclassifier.py +++ b/examples/xgbclassifier.py @@ -37,7 +37,7 @@ xgb, param_distributions=params, n_iter=3, - # early_stopping="MedianStoppingRule", + early_stopping="MedianStoppingRule", # use_gpu=True # Commented out for testing on travis, # but this is how you would use gpu ) From 6acf9effca3cdb9755420ec95cf2ae375ff3fccf Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Sat, 8 Aug 2020 05:29:35 -0700 Subject: [PATCH 13/47] latest ray mac --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0ba4fd0e..e16881a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ matrix: osx_image: xcode9.4 language: generic env: PYTHON=3.6 PYTHONWARNINGS=ignore OS=MAC SKLEARN_N_JOBS=1 + before_script: pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp36-cp36m-macosx_10_13_intel.whl - os: linux python: 3.7 env: PYTHON=3.7 OS=LINUX SKLEARN_N_JOBS=1 @@ -19,6 +20,7 @@ matrix: osx_image: xcode10.2 language: generic env: PYTHON=3.7 PYTHONWARNINGS=ignore OS=MAC SKLEARN_N_JOBS=1 + before_script: pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp37-cp37m-macosx_10_13_intel.whl - os: linux env: LINT=1 PYTHONWARNINGS=ignore # before_install: From d1e5ba6f75a09f3fa1e8870a979e0623d224ccfe Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Fri, 21 Aug 2020 11:22:33 -0700 Subject: [PATCH 14/47] remove ray wheels mac --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7dd7ff26..19eea1d5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,6 @@ matrix: osx_image: xcode9.4 language: generic env: PYTHON=3.6 PYTHONWARNINGS=ignore OS=MAC SKLEARN_N_JOBS=1 - before_script: pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp36-cp36m-macosx_10_13_intel.whl - os: linux python: 3.7 env: PYTHON=3.7 OS=LINUX SKLEARN_N_JOBS=1 @@ -20,7 +19,6 @@ matrix: osx_image: xcode10.2 language: generic env: PYTHON=3.7 PYTHONWARNINGS=ignore OS=MAC SKLEARN_N_JOBS=1 - before_script: pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp37-cp37m-macosx_10_13_intel.whl - os: linux env: LINT=1 PYTHONWARNINGS=ignore # before_install: From 4d40d18caf3e3840be4af68b1508ab7c7d61a02a Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Fri, 21 Aug 2020 11:40:32 -0700 Subject: [PATCH 15/47] build lgbm from github --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 19eea1d5..c4f34972 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,6 +37,7 @@ install: - pip3 install -q -r requirements.txt - pip3 install --upgrade -q keras - pip3 install -U -q scikit-learn pytest + - brew install cmake; brew install libomp; git clone --recursive https://github.com/microsoft/LightGBM ; cd LightGBM; mkdir build ; cd build; cmake ..; make -j4; cd .. # Nightly wheel installation # - pip3 install --pre --extra-index https://pypi.anaconda.org/scipy-wheels-nightly/simple -U scikit-learn From 4dda46c164d94cde2ad63bb4af5949d7def4fed9 Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Fri, 21 Aug 2020 12:11:40 -0700 Subject: [PATCH 16/47] fix travis path --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c4f34972..01666e44 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,7 @@ install: - pip3 install -q -r requirements.txt - pip3 install --upgrade -q keras - pip3 install -U -q scikit-learn pytest - - brew install cmake; brew install libomp; git clone --recursive https://github.com/microsoft/LightGBM ; cd LightGBM; mkdir build ; cd build; cmake ..; make -j4; cd .. + - brew install cmake; brew install libomp; git clone --recursive https://github.com/microsoft/LightGBM ; cd LightGBM; mkdir build ; cd build; cmake ..; make -j4; cd ../../ # Nightly wheel installation # - pip3 install --pre --extra-index https://pypi.anaconda.org/scipy-wheels-nightly/simple -U scikit-learn From d4dd6b70299ce5c1266d72888955ae5539b46d29 Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Sat, 22 Aug 2020 04:41:51 -0700 Subject: [PATCH 17/47] remove lgbm requirement --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c86879e4..943f2295 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ xgboost torch torchvision skorch -lightgbm +#lightgbm keras tensorflow click From 7852baa5cef26e8ddee9745431731338a59c05d9 Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Sat, 22 Aug 2020 22:29:12 -0700 Subject: [PATCH 18/47] line limits --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 01666e44..a6abfb91 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,8 @@ install: - pip3 install -q -r requirements.txt - pip3 install --upgrade -q keras - pip3 install -U -q scikit-learn pytest - - brew install cmake; brew install libomp; git clone --recursive https://github.com/microsoft/LightGBM ; cd LightGBM; mkdir build ; cd build; cmake ..; make -j4; cd ../../ + - brew install cmake; brew install libomp; git clone --recursive https://github.com/microsoft/LightGBM + - cd LightGBM; mkdir build ; cd build; cmake ..; make -j4; cd ../../ # Nightly wheel installation # - pip3 install --pre --extra-index https://pypi.anaconda.org/scipy-wheels-nightly/simple -U scikit-learn From 67ae04675f0cf8433acce084664b4b1c89e309e9 Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Sun, 23 Aug 2020 18:56:58 -0700 Subject: [PATCH 19/47] 1 boosting round --- tune_sklearn/_trainable.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tune_sklearn/_trainable.py b/tune_sklearn/_trainable.py index 7ce09ce2..07e219bc 100644 --- a/tune_sklearn/_trainable.py +++ b/tune_sklearn/_trainable.py @@ -52,16 +52,21 @@ def _setup(self, config): self.pickled = False if self.early_stopping: + self.is_lgbm = isinstance(self.estimator[0], LGBMModel) + self.is_xgb = isinstance(self.estimator[0], XGBModel) + + if self.is_xgb: + self.estimator_config["n_estimators"] = 1 + n_splits = self.cv.get_n_splits(self.X, self.y) self.fold_scores = np.zeros(n_splits) self.fold_train_scores = np.zeros(n_splits) for i in range(n_splits): self.estimator[i].set_params(**self.estimator_config) - self.is_lgbm = isinstance(self.estimator[0], LGBMModel) - self.is_xgb = isinstance(self.estimator[0], XGBModel) if self.is_lgbm or self.is_xgb: self.saved_models = [None for _ in range(n_splits)] + else: self.estimator.set_params(**self.estimator_config) From 5a9d66224e7557ffc8434bc2f228f290c4f2268e Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Sun, 23 Aug 2020 19:07:44 -0700 Subject: [PATCH 20/47] apply suggestions --- examples/warm_start.py | 5 +---- tune_sklearn/_trainable.py | 3 --- tune_sklearn/tune_basesearch.py | 15 +++++++++------ 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/examples/warm_start.py b/examples/warm_start.py index 655eb48b..a5a0d50e 100644 --- a/examples/warm_start.py +++ b/examples/warm_start.py @@ -11,7 +11,6 @@ from sklearn.model_selection import train_test_split from sklearn.datasets import load_iris from sklearn.linear_model import LogisticRegression -from ray.tune.schedulers import MedianStoppingRule import numpy as np x, y = load_iris(return_X_y=True) @@ -20,12 +19,10 @@ clf = LogisticRegression() parameter_grid = {"C": [1e-4, 1e-1, 1, 2]} -scheduler = MedianStoppingRule(grace_period=10.0) - tune_search = TuneGridSearchCV( clf, parameter_grid, - early_stopping=scheduler, + early_stopping="MedianStoppingRule", max_iters=10, ) tune_search.fit(x_train, y_train) diff --git a/tune_sklearn/_trainable.py b/tune_sklearn/_trainable.py index 7ac9aff0..43343b5b 100644 --- a/tune_sklearn/_trainable.py +++ b/tune_sklearn/_trainable.py @@ -11,8 +11,6 @@ from pickle import PicklingError import ray.cloudpickle as cpickle import warnings -from warnings import simplefilter -from sklearn.exceptions import ConvergenceWarning class _Trainable(Trainable): @@ -95,7 +93,6 @@ def _train(self): self.estimator[i].partial_fit(X_train, y_train, np.unique(self.y)) else: - simplefilter("ignore", category=ConvergenceWarning) self.estimator[i].fit(X_train, y_train) if self.return_train_score: diff --git a/tune_sklearn/tune_basesearch.py b/tune_sklearn/tune_basesearch.py index 62e846ba..889c47b7 100644 --- a/tune_sklearn/tune_basesearch.py +++ b/tune_sklearn/tune_basesearch.py @@ -406,12 +406,15 @@ def _can_early_stop(self): """ sklearn_module = ".".join( getattr(self.estimator, "__module__", None).split(".")[:-1]) - return (hasattr(self.estimator, "partial_fit") and callable( - getattr(self.estimator, "partial_fit", - None))) or (hasattr(self.estimator, "warm_start") - and hasattr(self.estimator, "max_iter") - and sklearn_module != "sklearn.tree" - and sklearn_module != "sklearn.ensemble") + + can_partial_fit = hasattr(self.estimator, "partial_fit") and callable( + getattr(self.estimator, "partial_fit", None)) + can_warm_start = (hasattr(self.estimator, "warm_start") + and hasattr(self.estimator, "max_iter") + and sklearn_module != "sklearn.tree" + and sklearn_module != "sklearn.ensemble") + + return can_partial_fit or can_warm_start def _fill_config_hyperparam(self, config): """Fill in the ``config`` dictionary with the hyperparameters. From 83051c414d1245c21353e213cf73ff2a1c1f8540 Mon Sep 17 00:00:00 2001 From: Michael Chau Date: Thu, 27 Aug 2020 15:03:04 -0700 Subject: [PATCH 21/47] apply suggestions --- examples/warm_start.py | 9 +++++++-- tune_sklearn/tune_basesearch.py | 10 +++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/examples/warm_start.py b/examples/warm_start.py index 2eadda9c..79803039 100644 --- a/examples/warm_start.py +++ b/examples/warm_start.py @@ -4,7 +4,12 @@ This example uses early stopping to further improve runtimes by eliminating worse hyperparameter choices early based off -of its average test score from cross validation. +of its average test score from cross validation. Usually +this will require the estimator to have `partial_fit`, but +we use sklearn's `warm_start` parameter to do this here. +We fit the estimator for one epoch, then `warm_start` +to pick up from where we left off, continuing until the +trial is early stopped or `max_iters` is reached. """ from tune_sklearn import TuneGridSearchCV @@ -22,7 +27,7 @@ tune_search = TuneGridSearchCV( clf, parameter_grid, - early_stopping="MedianStoppingRule", + early_stopping=True, max_iters=10, ) tune_search.fit(x_train, y_train) diff --git a/tune_sklearn/tune_basesearch.py b/tune_sklearn/tune_basesearch.py index 5b35dbd5..463cdb9d 100644 --- a/tune_sklearn/tune_basesearch.py +++ b/tune_sklearn/tune_basesearch.py @@ -427,19 +427,23 @@ def score(self, X, y=None): def _can_early_stop(self): """Helper method to determine if it is possible to do early stopping. - Only sklearn estimators with partial_fit can be early stopped. + Only sklearn estimators with `partial_fit` or `warm_start` can be early + stopped. warm_start works by picking up training from the previous + call to `fit`. Returns: bool: if the estimator can early stop """ - sklearn_module = ".".join( - getattr(self.estimator, "__module__", None).split(".")[:-1]) can_partial_fit = hasattr(self.estimator, "partial_fit") and callable( getattr(self.estimator, "partial_fit", None)) + + sklearn_module = ".".join( + getattr(self.estimator, "__module__", None).split(".")[:-1]) can_warm_start = (hasattr(self.estimator, "warm_start") and hasattr(self.estimator, "max_iter") + and sklearn_module is not None and sklearn_module != "sklearn.tree" and sklearn_module != "sklearn.ensemble") From 38726fe7dc0f12cbe3eb617485a116648c488399 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Fri, 28 Aug 2020 13:21:45 -0700 Subject: [PATCH 22/47] validation_fix --- tune_sklearn/tune_basesearch.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tune_sklearn/tune_basesearch.py b/tune_sklearn/tune_basesearch.py index 463cdb9d..14a633de 100644 --- a/tune_sklearn/tune_basesearch.py +++ b/tune_sklearn/tune_basesearch.py @@ -436,16 +436,20 @@ def _can_early_stop(self): """ - can_partial_fit = hasattr(self.estimator, "partial_fit") and callable( + from sklearn.tree import BaseDecisionTree + from sklearn.ensemble import BaseEnsemble + + can_partial_fit = callable( getattr(self.estimator, "partial_fit", None)) + is_not_tree_subclass = not issubclass( + type(self.estimator), BaseDecisionTree) + is_not_ensemble_subclass = not issubclass( + type(self.estimator), BaseEnsemble) - sklearn_module = ".".join( - getattr(self.estimator, "__module__", None).split(".")[:-1]) can_warm_start = (hasattr(self.estimator, "warm_start") and hasattr(self.estimator, "max_iter") - and sklearn_module is not None - and sklearn_module != "sklearn.tree" - and sklearn_module != "sklearn.ensemble") + and is_not_ensemble_subclass + and is_not_tree_subclass) return can_partial_fit or can_warm_start From 761c772dc79887a025885b41bdacf4804c6a637f Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Fri, 28 Aug 2020 14:47:37 -0700 Subject: [PATCH 23/47] improve --- tests/test_randomizedsearch.py | 33 +++++++++++++++++++++++++++++++++ tune_sklearn/_trainable.py | 4 ++++ tune_sklearn/tune_basesearch.py | 12 ++++++------ 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/tests/test_randomizedsearch.py b/tests/test_randomizedsearch.py index 00a78e76..e7724e60 100644 --- a/tests/test_randomizedsearch.py +++ b/tests/test_randomizedsearch.py @@ -134,6 +134,39 @@ def test_local_mode(self): tune_search.fit(x, y) self.assertTrue(wrapped_init.call_args[1]["local_mode"]) + def test_warm_start_detection(self): + parameter_grid = {"alpha": Real(1e-4, 1e-1, 1)} + from sklearn.ensemble import RandomForestClassifier + clf = RandomForestClassifier(max_depth=2, random_state=0) + tune_search = TuneSearchCV( + clf, + parameter_grid, + n_jobs=1, + max_iters=10, + local_dir="./test-result") + self.assertFalse(tune_search._can_early_stop()) + + from sklearn.tree import DecisionTreeClassifier + clf = DecisionTreeClassifier(random_state=0) + tune_search2 = TuneSearchCV( + clf, + parameter_grid, + n_jobs=1, + max_iters=10, + local_dir="./test-result") + self.assertFalse(tune_search2._can_early_stop()) + + from sklearn.linear_model import LogisticRegression + clf = LogisticRegression() + tune_search3 = TuneSearchCV( + clf, + parameter_grid, + n_jobs=1, + max_iters=10, + local_dir="./test-result") + + self.assertTrue(tune_search3._can_early_stop()) + if __name__ == "__main__": unittest.main() diff --git a/tune_sklearn/_trainable.py b/tune_sklearn/_trainable.py index d95a20cc..d9d332a9 100644 --- a/tune_sklearn/_trainable.py +++ b/tune_sklearn/_trainable.py @@ -58,6 +58,10 @@ def _setup(self, config): self.fold_scores = np.zeros(n_splits) self.fold_train_scores = np.zeros(n_splits) if not hasattr(self.estimator, "partial_fit"): + # max_iter here is different than the max_iters the user sets. + # max_iter is to make sklearn only fit for one epoch, + # while max_iters (which the user can set) is the usual max + # number of calls to _trainable. self.estimator_config["warm_start"] = True self.estimator_config["max_iter"] = 1 for i in range(n_splits): diff --git a/tune_sklearn/tune_basesearch.py b/tune_sklearn/tune_basesearch.py index 14a633de..9aff2acd 100644 --- a/tune_sklearn/tune_basesearch.py +++ b/tune_sklearn/tune_basesearch.py @@ -210,7 +210,11 @@ def __init__(self, self.estimator = estimator - if early_stopping and self._can_early_stop(): + if early_stopping and not self._can_early_stop(): + raise ValueError("Early stopping is not supported because " + "the estimator does not have `partial_fit` " + "or does not support warm_start.") + elif early_stopping and self._can_early_stop(): self.max_iters = max_iters if early_stopping is True: # Override the early_stopping variable so @@ -245,7 +249,7 @@ def __init__(self, else: raise TypeError("`early_stopping` must be a str, boolean, " "or tune scheduler") - elif not early_stopping: + else: warnings.warn("Early stopping is not enabled. " "To enable early stopping, pass in a supported " "scheduler from Tune and ensure the estimator " @@ -253,10 +257,6 @@ def __init__(self, self.max_iters = 1 self.early_stopping = None - else: - raise ValueError("Early stopping is not supported because " - "the estimator does not have `partial_fit`") - self.cv = cv self.scoring = scoring self.n_jobs = int(n_jobs or -1) From 7bd74b97ae1fad28e6528195878ebb3f7711cd97 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Fri, 28 Aug 2020 15:32:33 -0700 Subject: [PATCH 24/47] fix-resolution --- tests/test_randomizedsearch.py | 13 +++++ tune_sklearn/tune_basesearch.py | 86 +++++++++++++++++---------------- 2 files changed, 57 insertions(+), 42 deletions(-) diff --git a/tests/test_randomizedsearch.py b/tests/test_randomizedsearch.py index e7724e60..042bde4b 100644 --- a/tests/test_randomizedsearch.py +++ b/tests/test_randomizedsearch.py @@ -167,6 +167,19 @@ def test_warm_start_detection(self): self.assertTrue(tune_search3._can_early_stop()) + def test_warm_start_error(self): + parameter_grid = {"alpha": Real(1e-4, 1e-1, 1)} + from sklearn.ensemble import RandomForestClassifier + clf = RandomForestClassifier(max_depth=2, random_state=0) + tune_search = TuneSearchCV( + clf, + parameter_grid, + n_jobs=1, + warm_start=False, + max_iters=10, + local_dir="./test-result") + self.assertFalse(tune_search._can_early_stop()) + if __name__ == "__main__": unittest.main() diff --git a/tune_sklearn/tune_basesearch.py b/tune_sklearn/tune_basesearch.py index 9aff2acd..22d9c3f4 100644 --- a/tune_sklearn/tune_basesearch.py +++ b/tune_sklearn/tune_basesearch.py @@ -31,6 +31,33 @@ import os +def resolve_early_stopping(early_stopping, max_iters): + if isinstance(early_stopping, str): + if early_stopping in TuneBaseSearchCV.defined_schedulers: + if early_stopping == "PopulationBasedTraining": + return PopulationBasedTraining(metric="average_test_score") + elif early_stopping == "AsyncHyperBandScheduler": + return AsyncHyperBandScheduler( + metric="average_test_score", max_t=max_iters) + elif early_stopping == "HyperBandScheduler": + return HyperBandScheduler( + metric="average_test_score", max_t=max_iters) + elif early_stopping == "MedianStoppingRule": + return MedianStoppingRule(metric="average_test_score") + elif early_stopping == "ASHAScheduler": + return ASHAScheduler( + metric="average_test_score", max_t=max_iters) + raise ValueError("{} is not a defined scheduler. " + "Check the list of available schedulers." + .format(early_stopping)) + elif isinstance(early_stopping, TrialScheduler): + early_stopping.metric = "average_test_score" + return early_stopping + else: + raise TypeError("`early_stopping` must be a str, boolean, " + f"or tune scheduler. Got {type(early_stopping)}.") + + class TuneBaseSearchCV(BaseEstimator): """Abstract base class for TuneGridSearchCV and TuneSearchCV""" @@ -205,58 +232,33 @@ def __init__(self, error_score="raise", return_train_score=False, local_dir="~/ray_results", - max_iters=10, + max_iters=None, use_gpu=False): self.estimator = estimator - if early_stopping and not self._can_early_stop(): - raise ValueError("Early stopping is not supported because " - "the estimator does not have `partial_fit` " - "or does not support warm_start.") - elif early_stopping and self._can_early_stop(): - self.max_iters = max_iters - if early_stopping is True: + if not self._can_early_stop() and max_iters is not None: + warnings.warn("max_iters is set but incremental/partial training " + "is not enabled. To enable partial training, " + "ensure the estimator has `partial_fit` or " + "`warm_start`. Automatically setting max_iters=1.") + max_iters = 1 + self.max_iters = max_iters + + if early_stopping: + if not self._can_early_stop(): + raise ValueError("Early stopping is not supported because " + "the estimator does not have `partial_fit` " + "or does not support warm_start.") + elif early_stopping is True: # Override the early_stopping variable so # that it is resolved appropriately in # the next block early_stopping = "AsyncHyperBandScheduler" # Resolve the early stopping object - if isinstance(early_stopping, str): - if early_stopping in TuneBaseSearchCV.defined_schedulers: - if early_stopping == "PopulationBasedTraining": - self.early_stopping = PopulationBasedTraining( - metric="average_test_score") - elif early_stopping == "AsyncHyperBandScheduler": - self.early_stopping = AsyncHyperBandScheduler( - metric="average_test_score", max_t=max_iters) - elif early_stopping == "HyperBandScheduler": - self.early_stopping = HyperBandScheduler( - metric="average_test_score", max_t=max_iters) - elif early_stopping == "MedianStoppingRule": - self.early_stopping = MedianStoppingRule( - metric="average_test_score") - elif early_stopping == "ASHAScheduler": - self.early_stopping = ASHAScheduler( - metric="average_test_score", max_t=max_iters) - else: - raise ValueError("{} is not a defined scheduler. " - "Check the list of available schedulers." - .format(early_stopping)) - elif isinstance(early_stopping, TrialScheduler): - self.early_stopping = early_stopping - self.early_stopping.metric = "average_test_score" - else: - raise TypeError("`early_stopping` must be a str, boolean, " - "or tune scheduler") - else: - warnings.warn("Early stopping is not enabled. " - "To enable early stopping, pass in a supported " - "scheduler from Tune and ensure the estimator " - "has `partial_fit`.") + self.early_stopping = resolve_early_stopping( + early_stopping, self.max_iters) - self.max_iters = 1 - self.early_stopping = None self.cv = cv self.scoring = scoring self.n_jobs = int(n_jobs or -1) From 3a2b901847e152ad4248188d0076411b2e735ae4 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Fri, 28 Aug 2020 16:14:52 -0700 Subject: [PATCH 25/47] fix --- tests/test_randomizedsearch.py | 10 +++++++++- tune_sklearn/tune_search.py | 9 ++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/test_randomizedsearch.py b/tests/test_randomizedsearch.py index 042bde4b..dd64ddc1 100644 --- a/tests/test_randomizedsearch.py +++ b/tests/test_randomizedsearch.py @@ -175,10 +175,18 @@ def test_warm_start_error(self): clf, parameter_grid, n_jobs=1, - warm_start=False, + early_stopping=False, max_iters=10, local_dir="./test-result") self.assertFalse(tune_search._can_early_stop()) + with self.assertRaises(ValueError): + tune_search = TuneSearchCV( + clf, + parameter_grid, + n_jobs=1, + early_stopping=True, + max_iters=10, + local_dir="./test-result") if __name__ == "__main__": diff --git a/tune_sklearn/tune_search.py b/tune_sklearn/tune_search.py index ac60621e..fd3c1819 100644 --- a/tune_sklearn/tune_search.py +++ b/tune_sklearn/tune_search.py @@ -234,7 +234,7 @@ class TuneSearchCV(TuneBaseSearchCV): use_gpu (bool): Indicates whether to use gpu for fitting. Defaults to False. If True, training will use 1 gpu for `resources_per_trial`. - **kwargs (Any): + **search_kwargs (Any): Additional arguments to pass to the SearchAlgorithms (tune.suggest) objects. @@ -258,7 +258,7 @@ def __init__(self, max_iters=10, search_optimization="random", use_gpu=False, - **kwargs): + **search_kwargs): search_optimization = search_optimization.lower() available_optimizations = [ @@ -325,8 +325,11 @@ def __init__(self, self.num_samples = n_iter if search_optimization == "random": self.random_state = random_state + if search_kwargs: + raise ValueError("Random search does not support " + f"extra args: {search_kwargs}") self.search_optimization = search_optimization - self.kwargs = kwargs + self.search_kwargs = search_kwargs def _fill_config_hyperparam(self, config): """Fill in the ``config`` dictionary with the hyperparameters. From da2a042d0ffbc480257af7ce1c403eb8574cb476 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Fri, 28 Aug 2020 16:21:09 -0700 Subject: [PATCH 26/47] early --- examples/bohb_example.py | 3 ++- tune_sklearn/tune_basesearch.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/bohb_example.py b/examples/bohb_example.py index 04ce343b..6f95585e 100644 --- a/examples/bohb_example.py +++ b/examples/bohb_example.py @@ -22,8 +22,9 @@ bohb_tune_search = TuneSearchCV( SGDClassifier(), param_distributions=param_dists, - n_iter=2, + n_iter=20, max_iters=10, + verbose=2, search_optimization="bohb") bohb_tune_search.fit(X_train, y_train) diff --git a/tune_sklearn/tune_basesearch.py b/tune_sklearn/tune_basesearch.py index 22d9c3f4..001aab96 100644 --- a/tune_sklearn/tune_basesearch.py +++ b/tune_sklearn/tune_basesearch.py @@ -256,8 +256,9 @@ def __init__(self, # the next block early_stopping = "AsyncHyperBandScheduler" # Resolve the early stopping object - self.early_stopping = resolve_early_stopping( + early_stopping = resolve_early_stopping( early_stopping, self.max_iters) + self.early_stopping = early_stopping self.cv = cv self.scoring = scoring From 6bbe9776d2e11f0c5c596de2a07dd264dc4bc544 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Fri, 28 Aug 2020 16:21:58 -0700 Subject: [PATCH 27/47] Revert "early" This reverts commit da2a042d0ffbc480257af7ce1c403eb8574cb476. --- examples/bohb_example.py | 3 +-- tune_sklearn/tune_basesearch.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/bohb_example.py b/examples/bohb_example.py index 6f95585e..04ce343b 100644 --- a/examples/bohb_example.py +++ b/examples/bohb_example.py @@ -22,9 +22,8 @@ bohb_tune_search = TuneSearchCV( SGDClassifier(), param_distributions=param_dists, - n_iter=20, + n_iter=2, max_iters=10, - verbose=2, search_optimization="bohb") bohb_tune_search.fit(X_train, y_train) diff --git a/tune_sklearn/tune_basesearch.py b/tune_sklearn/tune_basesearch.py index 001aab96..22d9c3f4 100644 --- a/tune_sklearn/tune_basesearch.py +++ b/tune_sklearn/tune_basesearch.py @@ -256,9 +256,8 @@ def __init__(self, # the next block early_stopping = "AsyncHyperBandScheduler" # Resolve the early stopping object - early_stopping = resolve_early_stopping( + self.early_stopping = resolve_early_stopping( early_stopping, self.max_iters) - self.early_stopping = early_stopping self.cv = cv self.scoring = scoring From 3d39326cb6214f92b46c88990d2b4ffb6739a45c Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Fri, 28 Aug 2020 16:22:27 -0700 Subject: [PATCH 28/47] earlystop --- tune_sklearn/tune_basesearch.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tune_sklearn/tune_basesearch.py b/tune_sklearn/tune_basesearch.py index 22d9c3f4..ec0781d3 100644 --- a/tune_sklearn/tune_basesearch.py +++ b/tune_sklearn/tune_basesearch.py @@ -256,8 +256,9 @@ def __init__(self, # the next block early_stopping = "AsyncHyperBandScheduler" # Resolve the early stopping object - self.early_stopping = resolve_early_stopping( - early_stopping, self.max_iters) + early_stopping = resolve_early_stopping(early_stopping, + self.max_iters) + self.early_stopping = early_stopping self.cv = cv self.scoring = scoring From f14245ddbe5b33b70fc1089cdcfb1a8d859c732a Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Fri, 28 Aug 2020 16:25:08 -0700 Subject: [PATCH 29/47] n-trials --- README.md | 10 +++++----- examples/bayesian_sgd.py | 2 +- examples/bohb_example.py | 3 ++- examples/discrete_bayesian.py | 2 +- examples/discrete_hpbandster.py | 2 +- examples/discrete_hyperopt.py | 2 +- examples/hpbandster_sgd.py | 2 +- examples/hyperopt_sgd.py | 2 +- examples/lgbm.py | 2 +- examples/random_forest.py | 2 +- examples/xgbclassifier.py | 2 +- tests/test_gridsearch.py | 4 ++-- tests/test_randomizedsearch.py | 2 +- tune_sklearn/tune_search.py | 12 ++++++------ 14 files changed, 25 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 47211552..4d298637 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # tune-sklearn [![Build Status](https://travis-ci.com/ray-project/tune-sklearn.svg?branch=master)](https://travis-ci.com/ray-project/tune-sklearn) -Tune-sklearn is a package that integrates Ray Tune's hyperparameter tuning and scikit-learn's models, allowing users to optimize hyerparameter searching for sklearn using Tune's schedulers (more details in the [Tune Documentation](http://tune.io/)). +Tune-sklearn is a package that integrates Ray Tune's hyperparameter tuning and scikit-learn's models, allowing users to optimize hyerparameter searching for sklearn using Tune's schedulers (more details in the [Tune Documentation](http://tune.io/)). Tune-sklearn follows the same API as scikit-learn's GridSearchCV, but allows for more flexibility in defining hyperparameter search regions, such as distributions to sample from. -Tune-sklearn provides additional benefits if specifying a scheduler **with an estimator that supports early stopping**. The list of estimators that can be supported from scikit-learn can be found in [scikit-learn's documentation at section 8.1.1.3](https://scikit-learn.org/stable/modules/computing.html#strategies-to-scale-computationally-bigger-data). +Tune-sklearn provides additional benefits if specifying a scheduler **with an estimator that supports early stopping**. The list of estimators that can be supported from scikit-learn can be found in [scikit-learn's documentation at section 8.1.1.3](https://scikit-learn.org/stable/modules/computing.html#strategies-to-scale-computationally-bigger-data). If the estimator does not support `partial_fit`, a warning will be shown saying early stopping cannot be done and it will simply run the cross-validation on Ray's parallel back-end. @@ -70,7 +70,7 @@ from sklearn.model_selection import GridSearchCV # n_jobs=-1 enables use of all cores like Tune does sklearn_search = GridSearchCV( SGDClassifier(), - parameters, + parameters, n_jobs=-1 ) @@ -120,7 +120,7 @@ param_dists = { bohb_tune_search = TuneSearchCV(SGDClassifier(), param_distributions=param_dists, - n_iter=2, + n_trials=2, max_iters=10, search_optimization="bohb" ) @@ -129,7 +129,7 @@ bohb_tune_search.fit(X_train, y_train) hyperopt_tune_search = TuneSearchCV(SGDClassifier(), param_distributions=param_dists, - n_iter=2, + n_trials=2, early_stopping=True, # uses ASHAScheduler if set to True max_iters=10, search_optimization="hyperopt" diff --git a/examples/bayesian_sgd.py b/examples/bayesian_sgd.py index e4e6d053..307e3046 100644 --- a/examples/bayesian_sgd.py +++ b/examples/bayesian_sgd.py @@ -19,7 +19,7 @@ clf, parameter_grid, search_optimization="bayesian", - n_iter=3, + n_trials=3, early_stopping=scheduler, max_iters=10) tune_search.fit(x_train, y_train) diff --git a/examples/bohb_example.py b/examples/bohb_example.py index 04ce343b..9c9f02a3 100644 --- a/examples/bohb_example.py +++ b/examples/bohb_example.py @@ -22,8 +22,9 @@ bohb_tune_search = TuneSearchCV( SGDClassifier(), param_distributions=param_dists, - n_iter=2, + n_trials=20, max_iters=10, + verbose=2, search_optimization="bohb") bohb_tune_search.fit(X_train, y_train) diff --git a/examples/discrete_bayesian.py b/examples/discrete_bayesian.py index 12805ad6..d2a58d17 100644 --- a/examples/discrete_bayesian.py +++ b/examples/discrete_bayesian.py @@ -16,7 +16,7 @@ space, n_jobs=3, search_optimization="hyperopt", - n_iter=3, + n_trials=3, max_iters=10) tune_search.fit(X_train, y_train) diff --git a/examples/discrete_hpbandster.py b/examples/discrete_hpbandster.py index b874792f..9d4fd6e7 100644 --- a/examples/discrete_hpbandster.py +++ b/examples/discrete_hpbandster.py @@ -21,7 +21,7 @@ RandomForestClassifier(), space, search_optimization="bohb", - n_iter=3, + n_trials=3, max_iters=10) tune_search.fit(X_train, y_train) diff --git a/examples/discrete_hyperopt.py b/examples/discrete_hyperopt.py index 68d26e64..006d67ad 100644 --- a/examples/discrete_hyperopt.py +++ b/examples/discrete_hyperopt.py @@ -20,7 +20,7 @@ RandomForestClassifier(), space, search_optimization="hyperopt", - n_iter=3, + n_trials=3, max_iters=10) tune_search.fit(X_train, y_train) diff --git a/examples/hpbandster_sgd.py b/examples/hpbandster_sgd.py index f0cb5343..39f70bf8 100644 --- a/examples/hpbandster_sgd.py +++ b/examples/hpbandster_sgd.py @@ -13,7 +13,7 @@ parameter_grid = {"alpha": (1e-4, 1), "epsilon": (0.01, 0.1)} tune_search = TuneSearchCV( - clf, parameter_grid, search_optimization="bohb", n_iter=3, max_iters=10) + clf, parameter_grid, search_optimization="bohb", n_trials=3, max_iters=10) tune_search.fit(x_train, y_train) pred = tune_search.predict(x_test) diff --git a/examples/hyperopt_sgd.py b/examples/hyperopt_sgd.py index 0dc9d6af..278adcf4 100644 --- a/examples/hyperopt_sgd.py +++ b/examples/hyperopt_sgd.py @@ -19,7 +19,7 @@ clf, parameter_grid, search_optimization="hyperopt", - n_iter=3, + n_trials=3, early_stopping=scheduler, max_iters=10) tune_search.fit(x_train, y_train) diff --git a/examples/lgbm.py b/examples/lgbm.py index d4a2d9cd..f66799b3 100644 --- a/examples/lgbm.py +++ b/examples/lgbm.py @@ -29,7 +29,7 @@ "subsample_freq": [20] } -gs = TuneSearchCV(model, param_dists, n_iter=5, scoring="accuracy") +gs = TuneSearchCV(model, param_dists, n_trials=5, scoring="accuracy") gs.fit(X_train, y_train) print(gs.cv_results_) diff --git a/examples/random_forest.py b/examples/random_forest.py index 8c6cf86d..6a950432 100644 --- a/examples/random_forest.py +++ b/examples/random_forest.py @@ -21,7 +21,7 @@ "max_depth": randint(2, 10) } -tune_search = TuneSearchCV(clf, param_distributions, n_iter=3) +tune_search = TuneSearchCV(clf, param_distributions, n_trials=3) tune_search.fit(x_train, y_train) diff --git a/examples/xgbclassifier.py b/examples/xgbclassifier.py index 8b31f5e3..22024005 100644 --- a/examples/xgbclassifier.py +++ b/examples/xgbclassifier.py @@ -36,7 +36,7 @@ digit_search = TuneSearchCV( xgb, param_distributions=params, - n_iter=3, + n_trials=3, # use_gpu=True # Commented out for testing on travis, # but this is how you would use gpu ) diff --git a/tests/test_gridsearch.py b/tests/test_gridsearch.py index 2fe7e35f..0cd530ae 100644 --- a/tests/test_gridsearch.py +++ b/tests/test_gridsearch.py @@ -139,7 +139,7 @@ def test_grid_search_no_score(self): @parameterized.expand([("grid", TuneGridSearchCV, {}), ("random", TuneSearchCV, { - "n_iter": 1 + "n_trials": 1 })]) def test_hyperparameter_searcher_with_fit_params(self, name, cls, kwargs): X = np.arange(100).reshape(10, 10) @@ -261,7 +261,7 @@ def test_trivial_cv_results_attr(self): grid_search.fit(X, y) self.assertTrue(hasattr(grid_search, "cv_results_")) - random_search = TuneSearchCV(clf, {"foo_param": [0]}, n_iter=1, cv=3) + random_search = TuneSearchCV(clf, {"foo_param": [0]}, n_trials=1, cv=3) random_search.fit(X, y) self.assertTrue(hasattr(random_search, "cv_results_")) diff --git a/tests/test_randomizedsearch.py b/tests/test_randomizedsearch.py index dd64ddc1..68282625 100644 --- a/tests/test_randomizedsearch.py +++ b/tests/test_randomizedsearch.py @@ -28,7 +28,7 @@ def test_random_search_cv_results(self): params = dict(C=expon(scale=10), gamma=expon(scale=0.1)) random_search = TuneSearchCV( SVC(), - n_iter=n_search_iter, + n_trials=n_search_iter, cv=n_splits, param_distributions=params, return_train_score=True, diff --git a/tune_sklearn/tune_search.py b/tune_sklearn/tune_search.py index fd3c1819..813c35ed 100644 --- a/tune_sklearn/tune_search.py +++ b/tune_sklearn/tune_search.py @@ -78,7 +78,7 @@ class TuneSearchCV(TuneBaseSearchCV): In contrast to GridSearchCV, not all parameter values are tried out, but rather a fixed number of parameter settings is sampled from the specified distributions. The number of parameter settings that are tried is - given by n_iter. + given by n_trials. Args: estimator (`estimator`): This is assumed to implement the @@ -143,8 +143,8 @@ class TuneSearchCV(TuneBaseSearchCV): this parameter is ignored for ``"bohb"``, as it requires ``HyperBandForBOHB``. - n_iter (int): Number of parameter settings that are sampled. - n_iter trades off runtime vs quality of the solution. + n_trials (int): Number of parameter settings that are sampled. + n_trials trades off runtime vs quality of the solution. Defaults to 10. scoring (str, `callable`, or None): A single string or a callable to evaluate the predictions on the test set. @@ -213,7 +213,7 @@ class TuneSearchCV(TuneBaseSearchCV): local_dir (str): A string that defines where checkpoints will be stored. Defaults to "~/ray_results" max_iters (int): Indicates the maximum number of epochs to run for each - hyperparameter configuration sampled (specified by ``n_iter``). + hyperparameter configuration sampled (specified by ``n_trials``). This parameter is used for early stopping. Defaults to 10. search_optimization ("random" or "bayesian" or "bohb" or "hyperopt"): Randomized search is invoked with ``search_optimization`` set to @@ -244,7 +244,7 @@ def __init__(self, estimator, param_distributions, early_stopping=None, - n_iter=10, + n_trials=10, scoring=None, n_jobs=None, sk_n_jobs=-1, @@ -322,7 +322,7 @@ def __init__(self, use_gpu=use_gpu) self.param_distributions = param_distributions - self.num_samples = n_iter + self.num_samples = n_trials if search_optimization == "random": self.random_state = random_state if search_kwargs: From 17ab13e6a2f24c689a00a5971aa07d84334b5d0c Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Fri, 28 Aug 2020 16:26:07 -0700 Subject: [PATCH 30/47] fix --- examples/bohb_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/bohb_example.py b/examples/bohb_example.py index 9c9f02a3..520af907 100644 --- a/examples/bohb_example.py +++ b/examples/bohb_example.py @@ -23,7 +23,7 @@ SGDClassifier(), param_distributions=param_dists, n_trials=20, - max_iters=10, + max_iters=20, verbose=2, search_optimization="bohb") From d5506543d43b2098b5ad370344686c1486c789d3 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Fri, 28 Aug 2020 16:26:42 -0700 Subject: [PATCH 31/47] fix --- tune_sklearn/tune_search.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tune_sklearn/tune_search.py b/tune_sklearn/tune_search.py index fd3c1819..891255ad 100644 --- a/tune_sklearn/tune_search.py +++ b/tune_sklearn/tune_search.py @@ -581,7 +581,7 @@ def _tune_run(self, config, resources_per_trial): Optimizer(spaces), hyperparameter_names, metric="average_test_score", - **self.kwargs) + **self.search_kwargs) elif self.search_optimization == "bohb": from ray.tune.suggest.bohb import TuneBOHB @@ -590,7 +590,7 @@ def _tune_run(self, config, resources_per_trial): config_space, metric="average_test_score", mode="max", - **self.kwargs) + **self.search_kwargs) elif self.search_optimization == "optuna": from ray.tune.suggest.optuna import OptunaSearch @@ -599,7 +599,7 @@ def _tune_run(self, config, resources_per_trial): config_space, metric="average_test_score", mode="max", - **self.kwargs) + **self.search_kwargs) elif self.search_optimization == "hyperopt": from ray.tune.suggest.hyperopt import HyperOptSearch @@ -608,7 +608,7 @@ def _tune_run(self, config, resources_per_trial): config_space, metric="average_test_score", mode="max", - **self.kwargs) + **self.search_kwargs) if isinstance(self.n_jobs, int) and self.n_jobs > 0: search_algo = ConcurrencyLimiter( From 2c2044a006fadbc3da9fa62e60075f14a0b6439b Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Fri, 28 Aug 2020 17:02:16 -0700 Subject: [PATCH 32/47] Fix up early stopping --- examples/bohb_example.py | 7 +++-- tune_sklearn/_trainable.py | 53 +++++++++++++++++---------------- tune_sklearn/tune_gridsearch.py | 4 +-- tune_sklearn/tune_search.py | 9 +++--- 4 files changed, 37 insertions(+), 36 deletions(-) diff --git a/examples/bohb_example.py b/examples/bohb_example.py index 520af907..0847480a 100644 --- a/examples/bohb_example.py +++ b/examples/bohb_example.py @@ -22,9 +22,10 @@ bohb_tune_search = TuneSearchCV( SGDClassifier(), param_distributions=param_dists, - n_trials=20, - max_iters=20, + n_trials=1, + max_iters=100, verbose=2, - search_optimization="bohb") + search_optimization="bohb", +) bohb_tune_search.fit(X_train, y_train) diff --git a/tune_sklearn/_trainable.py b/tune_sklearn/_trainable.py index d9d332a9..16945fc6 100644 --- a/tune_sklearn/_trainable.py +++ b/tune_sklearn/_trainable.py @@ -20,6 +20,14 @@ class _Trainable(Trainable): and restore routines. """ + estimator_list = None + + def _can_partial_fit(self): + return hasattr(self.main_estimator, "partial_fit") + + @property + def main_estimator(self): + return self.estimator_list[0] def setup(self, config): # forward-compatbility @@ -36,7 +44,7 @@ def _setup(self, config): stopping if it is set to true. """ - self.estimator = clone(config.pop("estimator")) + self.estimator_list = clone(config.pop("estimator_list")) self.early_stopping = config.pop("early_stopping") X_id = config.pop("X_id") self.X = ray.get(X_id) @@ -57,7 +65,7 @@ def _setup(self, config): n_splits = self.cv.get_n_splits(self.X, self.y) self.fold_scores = np.zeros(n_splits) self.fold_train_scores = np.zeros(n_splits) - if not hasattr(self.estimator, "partial_fit"): + if not self._can_partial_fit(): # max_iter here is different than the max_iters the user sets. # max_iter is to make sklearn only fit for one epoch, # while max_iters (which the user can set) is the usual max @@ -65,9 +73,9 @@ def _setup(self, config): self.estimator_config["warm_start"] = True self.estimator_config["max_iter"] = 1 for i in range(n_splits): - self.estimator[i].set_params(**self.estimator_config) + self.estimator_list[i].set_params(**self.estimator_config) else: - self.estimator.set_params(**self.estimator_config) + self.main_estimator.set_params(**self.estimator_config) def step(self): # forward-compatbility @@ -95,25 +103,25 @@ def _train(self): """ if self.early_stopping: for i, (train, test) in enumerate(self.cv.split(self.X, self.y)): - X_train, y_train = _safe_split(self.estimator[i], self.X, + X_train, y_train = _safe_split(self.estimator_list[i], self.X, self.y, train) X_test, y_test = _safe_split( - self.estimator[i], + self.estimator_list[i], self.X, self.y, test, train_indices=train) - if hasattr(self.estimator, "partial_fit"): - self.estimator[i].partial_fit(X_train, y_train, - np.unique(self.y)) + if self._can_partial_fit(): + self.estimator_list[i].partial_fit(X_train, y_train, + np.unique(self.y)) else: - self.estimator[i].fit(X_train, y_train) + self.estimator_list[i].fit(X_train, y_train) if self.return_train_score: self.fold_train_scores[i] = self.scoring( - self.estimator[i], X_train, y_train) - self.fold_scores[i] = self.scoring(self.estimator[i], X_test, - y_test) + self.estimator_list[i], X_train, y_train) + self.fold_scores[i] = self.scoring(self.estimator_list[i], + X_test, y_test) ret = {} total = 0 @@ -137,7 +145,7 @@ def _train(self): else: try: scores = cross_validate( - self.estimator, + self.main_estimator, self.X, self.y, cv=self.cv, @@ -152,7 +160,7 @@ def _train(self): "validation. Proceeding to cross validate with " "one core.") scores = cross_validate( - self.estimator, + self.main_estimator, self.X, self.y, cv=self.cv, @@ -196,14 +204,7 @@ def _save(self, checkpoint_dir): """ path = os.path.join(checkpoint_dir, "checkpoint") with open(path, "wb") as f: - try: - cpickle.dump(self.estimator, f) - self.pickled = True - except PicklingError: - self.pickled = False - warnings.warn("{} could not be pickled. " - "Restoring estimators may run into issues." - .format(self.estimator)) + cpickle.dump(self.estimator_list, f) return path def load_checkpoint(self, checkpoint): @@ -217,10 +218,10 @@ def _restore(self, checkpoint): checkpoint (str): file path to pickled checkpoint file. """ - if self.pickled: + try: with open(checkpoint, "rb") as f: - self.estimator = cpickle.load(f) - else: + self.estimator_list = cpickle.load(f) + except Exception: warnings.warn("No estimator restored") def reset_config(self, new_config): diff --git a/tune_sklearn/tune_gridsearch.py b/tune_sklearn/tune_gridsearch.py index 5f3d3cdc..b8d9003a 100644 --- a/tune_sklearn/tune_gridsearch.py +++ b/tune_sklearn/tune_gridsearch.py @@ -189,11 +189,11 @@ def _tune_run(self, config, resources_per_trial): """ if self.early_stopping is not None: - config["estimator"] = [ + config["estimator_list"] = [ clone(self.estimator) for _ in range(self.n_splits) ] else: - config["estimator"] = self.estimator + config["estimator_list"] = [self.estimator] if isinstance(self.param_grid, list): analysis = tune.run( diff --git a/tune_sklearn/tune_search.py b/tune_sklearn/tune_search.py index e4a28911..a6d99c5d 100644 --- a/tune_sklearn/tune_search.py +++ b/tune_sklearn/tune_search.py @@ -301,8 +301,7 @@ def __init__(self, if search_optimization == "bohb": from ray.tune.schedulers import HyperBandForBOHB - if early_stopping and not isinstance(early_stopping, - HyperBandForBOHB): + if not isinstance(early_stopping, HyperBandForBOHB): early_stopping = HyperBandForBOHB( metric="average_test_score", max_t=max_iters) @@ -526,7 +525,7 @@ def _try_import_required_libraries(self, search_optimization): def _tune_run(self, config, resources_per_trial): """Wrapper to call ``tune.run``. Multiple estimators are generated when early stopping is possible, whereas a single estimator is - generated when early stopping is not possible. + generated when early stopping is not possible. Args: config (dict): Configurations such as hyperparameters to run @@ -542,7 +541,7 @@ def _tune_run(self, config, resources_per_trial): """ stop_condition = {"training_iteration": self.max_iters} if self.early_stopping is not None: - config["estimator"] = [ + config["estimator_list"] = [ clone(self.estimator) for _ in range(self.n_splits) ] if hasattr(self.early_stopping, "_max_t_attr"): @@ -551,7 +550,7 @@ def _tune_run(self, config, resources_per_trial): # the solution is to make the stop condition very big stop_condition = {"training_iteration": self.max_iters * 10} else: - config["estimator"] = self.estimator + config["estimator_list"] = [self.estimator] if self.search_optimization == "random": run_args = dict( From c87122066e31140ad390b2119ba5c1adfc5dca53 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Fri, 28 Aug 2020 21:09:26 -0700 Subject: [PATCH 33/47] fixup-search --- examples/bohb_example.py | 2 +- tune_sklearn/tune_basesearch.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/bohb_example.py b/examples/bohb_example.py index 0847480a..4f3a6876 100644 --- a/examples/bohb_example.py +++ b/examples/bohb_example.py @@ -22,7 +22,7 @@ bohb_tune_search = TuneSearchCV( SGDClassifier(), param_distributions=param_dists, - n_trials=1, + n_trials=20, max_iters=100, verbose=2, search_optimization="bohb", diff --git a/tune_sklearn/tune_basesearch.py b/tune_sklearn/tune_basesearch.py index a3323016..d8b9142e 100644 --- a/tune_sklearn/tune_basesearch.py +++ b/tune_sklearn/tune_basesearch.py @@ -503,9 +503,9 @@ def _clean_config_dict(self, config): and the values are the numeric values set to those variables. """ for key in [ - "estimator", "early_stopping", "X_id", "y_id", "groups", "cv", - "fit_params", "scoring", "max_iters", "return_train_score", - "n_jobs" + "estimator_list", "early_stopping", "X_id", "y_id", + "groups", "cv", "fit_params", "scoring", "max_iters", + "return_train_score", "n_jobs" ]: config.pop(key, None) return config From 7f15c25b31b1f777528b1888c65a21b840c73bf1 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Fri, 28 Aug 2020 21:48:08 -0700 Subject: [PATCH 34/47] fix --- examples/discrete_hpbandster.py | 4 ++-- tune_sklearn/tune_basesearch.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/discrete_hpbandster.py b/examples/discrete_hpbandster.py index 9d4fd6e7..6ce32b80 100644 --- a/examples/discrete_hpbandster.py +++ b/examples/discrete_hpbandster.py @@ -1,7 +1,7 @@ from tune_sklearn import TuneSearchCV from sklearn import datasets from sklearn.model_selection import train_test_split -from sklearn.ensemble import RandomForestClassifier +from sklearn.linear_model import SGDClassifier import ConfigSpace as CS digits = datasets.load_digits() @@ -18,7 +18,7 @@ } tune_search = TuneSearchCV( - RandomForestClassifier(), + SGDClassifier(), space, search_optimization="bohb", n_trials=3, diff --git a/tune_sklearn/tune_basesearch.py b/tune_sklearn/tune_basesearch.py index d8b9142e..abb3b448 100644 --- a/tune_sklearn/tune_basesearch.py +++ b/tune_sklearn/tune_basesearch.py @@ -503,8 +503,8 @@ def _clean_config_dict(self, config): and the values are the numeric values set to those variables. """ for key in [ - "estimator_list", "early_stopping", "X_id", "y_id", - "groups", "cv", "fit_params", "scoring", "max_iters", + "estimator_list", "early_stopping", "X_id", "y_id", "groups", + "cv", "fit_params", "scoring", "max_iters", "return_train_score", "n_jobs" ]: config.pop(key, None) From 806a36f1323b7a82748982160dc827d1a4449f00 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Fri, 28 Aug 2020 21:49:58 -0700 Subject: [PATCH 35/47] fix --- examples/bohb_example.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/bohb_example.py b/examples/bohb_example.py index 4f3a6876..99b88158 100644 --- a/examples/bohb_example.py +++ b/examples/bohb_example.py @@ -24,7 +24,6 @@ param_distributions=param_dists, n_trials=20, max_iters=100, - verbose=2, search_optimization="bohb", ) From 23da33a217d4ec0763319159f26e99a0d57f3d0d Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sat, 29 Aug 2020 00:27:33 -0700 Subject: [PATCH 36/47] remove due to lack of support --- examples/discrete_hpbandster.py | 29 ----------------------------- tune_sklearn/tune_basesearch.py | 3 ++- 2 files changed, 2 insertions(+), 30 deletions(-) delete mode 100644 examples/discrete_hpbandster.py diff --git a/examples/discrete_hpbandster.py b/examples/discrete_hpbandster.py deleted file mode 100644 index 6ce32b80..00000000 --- a/examples/discrete_hpbandster.py +++ /dev/null @@ -1,29 +0,0 @@ -from tune_sklearn import TuneSearchCV -from sklearn import datasets -from sklearn.model_selection import train_test_split -from sklearn.linear_model import SGDClassifier -import ConfigSpace as CS - -digits = datasets.load_digits() -X = digits.data -y = digits.target - -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) - -space = { - "n_estimators": CS.UniformIntegerHyperparameter("n_estimators", 100, 200), - "min_weight_fraction_leaf": (0.0, 0.5), - "min_samples_leaf": CS.UniformIntegerHyperparameter( - "min_samples_leaf", 1, 5) -} - -tune_search = TuneSearchCV( - SGDClassifier(), - space, - search_optimization="bohb", - n_trials=3, - max_iters=10) -tune_search.fit(X_train, y_train) - -print(tune_search.cv_results_) -print(tune_search.best_params_) diff --git a/tune_sklearn/tune_basesearch.py b/tune_sklearn/tune_basesearch.py index abb3b448..ceb31beb 100644 --- a/tune_sklearn/tune_basesearch.py +++ b/tune_sklearn/tune_basesearch.py @@ -249,7 +249,8 @@ def __init__(self, if not self._can_early_stop(): raise ValueError("Early stopping is not supported because " "the estimator does not have `partial_fit` " - "or does not support warm_start.") + ", does not support warm_start, or is a " + "tree or ensemble classifier.") elif early_stopping is True: # Override the early_stopping variable so # that it is resolved appropriately in From ab478936c71fc2672cc36482a8beaf8f96b240bd Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sat, 29 Aug 2020 01:25:13 -0700 Subject: [PATCH 37/47] fix --- tune_sklearn/_trainable.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tune_sklearn/_trainable.py b/tune_sklearn/_trainable.py index 16945fc6..0929d190 100644 --- a/tune_sklearn/_trainable.py +++ b/tune_sklearn/_trainable.py @@ -203,8 +203,11 @@ def _save(self, checkpoint_dir): """ path = os.path.join(checkpoint_dir, "checkpoint") - with open(path, "wb") as f: - cpickle.dump(self.estimator_list, f) + try: + with open(path, "wb") as f: + cpickle.dump(self.estimator_list, f) + except Exception as e: + warnings.warn("Unable to save estimator.") return path def load_checkpoint(self, checkpoint): From 78b5394294d6f8aefc78f3cbf1691db94af724bc Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sat, 29 Aug 2020 13:11:11 -0700 Subject: [PATCH 38/47] execption --- tune_sklearn/_trainable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tune_sklearn/_trainable.py b/tune_sklearn/_trainable.py index 0929d190..9a9dd435 100644 --- a/tune_sklearn/_trainable.py +++ b/tune_sklearn/_trainable.py @@ -206,7 +206,7 @@ def _save(self, checkpoint_dir): try: with open(path, "wb") as f: cpickle.dump(self.estimator_list, f) - except Exception as e: + except Exception: warnings.warn("Unable to save estimator.") return path From 51af043f9ea0d7c768bbea520b5ec3510708e5ef Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sat, 29 Aug 2020 17:47:29 -0700 Subject: [PATCH 39/47] fix-xgboost --- tune_sklearn/_trainable.py | 15 +++++++++++++-- tune_sklearn/tune_search.py | 4 ---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tune_sklearn/_trainable.py b/tune_sklearn/_trainable.py index 4f85a787..3e40343a 100644 --- a/tune_sklearn/_trainable.py +++ b/tune_sklearn/_trainable.py @@ -14,6 +14,14 @@ import warnings +def try_import_xgboost(): + try: + import xgboost # ignore: F401 + return True + except ImportError: + return False + + class _Trainable(Trainable): """Class to be passed in as the first argument of tune.run to train models. @@ -32,8 +40,10 @@ def main_estimator(self): @property def is_xgb(self): - from xgboost.sklearn import XGBModel - return isinstance(self.main_estimator, XGBModel) + if try_import_xgboost(): + from xgboost.sklearn import XGBModel + return isinstance(self.main_estimator, XGBModel) + return False def setup(self, config): @@ -70,6 +80,7 @@ def _setup(self, config): if self.early_stopping: if self.is_xgb: + # Does this work? self.estimator_config["n_estimators"] = 1 n_splits = self.cv.get_n_splits(self.X, self.y) diff --git a/tune_sklearn/tune_search.py b/tune_sklearn/tune_search.py index a6d99c5d..8284c67d 100644 --- a/tune_sklearn/tune_search.py +++ b/tune_sklearn/tune_search.py @@ -490,7 +490,6 @@ def _try_import_required_libraries(self, search_optimization): from skopt import Optimizer # noqa: F401 from ray.tune.suggest.skopt import SkOptSearch # noqa: F401 except ImportError: - logger.exception() raise ImportError( "It appears that scikit-optimize is not installed. " "Do: pip install scikit-optimize") from None @@ -500,7 +499,6 @@ def _try_import_required_libraries(self, search_optimization): from ray.tune.schedulers import HyperBandForBOHB # noqa: F401 import ConfigSpace as CS # noqa: F401 except ImportError: - logger.exception() raise ImportError( "It appears that either HpBandSter or ConfigSpace " "is not installed. " @@ -510,7 +508,6 @@ def _try_import_required_libraries(self, search_optimization): from ray.tune.suggest.hyperopt import HyperOptSearch # noqa: F401,E501 from hyperopt import hp # noqa: F401 except ImportError: - logger.exception() raise ImportError("It appears that hyperopt is not installed. " "Do: pip install hyperopt") from None elif search_optimization == "optuna": @@ -518,7 +515,6 @@ def _try_import_required_libraries(self, search_optimization): from ray.tune.suggest.optuna import OptunaSearch, param # noqa: F401,E501 import optuna # noqa: F401 except ImportError: - logger.exception() raise ImportError("It appears that optuna is not installed. " "Do: pip install optuna") from None From e81dcd29e72e0ac95e1823c768fabc5302e05978 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sat, 29 Aug 2020 17:56:25 -0700 Subject: [PATCH 40/47] requirements --- .travis.yml | 2 -- requirements.txt => requirements-test.txt | 2 +- tune_sklearn/_trainable.py | 4 ---- 3 files changed, 1 insertion(+), 7 deletions(-) rename requirements.txt => requirements-test.txt (93%) diff --git a/.travis.yml b/.travis.yml index f8f71232..bfdd53ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,8 +37,6 @@ install: - pip3 install -q -r requirements.txt - pip3 install --upgrade -q keras - pip3 install -U -q scikit-learn pytest - - brew install cmake; brew install libomp; git clone --recursive https://github.com/microsoft/LightGBM - - cd LightGBM; mkdir build ; cd build; cmake ..; make -j4; cd ../../ # Nightly wheel installation # - pip3 install --pre --extra-index https://pypi.anaconda.org/scipy-wheels-nightly/simple -U scikit-learn diff --git a/requirements.txt b/requirements-test.txt similarity index 93% rename from requirements.txt rename to requirements-test.txt index 943f2295..c86879e4 100644 --- a/requirements.txt +++ b/requirements-test.txt @@ -10,7 +10,7 @@ xgboost torch torchvision skorch -#lightgbm +lightgbm keras tensorflow click diff --git a/tune_sklearn/_trainable.py b/tune_sklearn/_trainable.py index 7cdcd147..dffa37a9 100644 --- a/tune_sklearn/_trainable.py +++ b/tune_sklearn/_trainable.py @@ -78,10 +78,6 @@ def _setup(self, config): self.pickled = False if self.early_stopping: - if self.is_xgb: - # Does this work? - self.estimator_config["n_estimators"] = 1 - n_splits = self.cv.get_n_splits(self.X, self.y) self.fold_scores = np.zeros(n_splits) self.fold_train_scores = np.zeros(n_splits) From e0308511a989f7598ff44139a0bdcc9eaa1888f4 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sat, 29 Aug 2020 17:59:57 -0700 Subject: [PATCH 41/47] fix --- tune_sklearn/_trainable.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tune_sklearn/_trainable.py b/tune_sklearn/_trainable.py index dffa37a9..3aea4775 100644 --- a/tune_sklearn/_trainable.py +++ b/tune_sklearn/_trainable.py @@ -6,7 +6,6 @@ from sklearn.base import clone from sklearn.model_selection import cross_validate from sklearn.utils.metaestimators import _safe_split -from lightgbm import LGBMModel import numpy as np import os from pickle import PicklingError From b69dc890159fb4afa7d7e321b125c54cc195a293 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sat, 29 Aug 2020 18:05:06 -0700 Subject: [PATCH 42/47] fix --- examples/lgbm.py | 2 +- tune_sklearn/_trainable.py | 24 ++++++------------------ tune_sklearn/tune_basesearch.py | 7 +++---- 3 files changed, 10 insertions(+), 23 deletions(-) diff --git a/examples/lgbm.py b/examples/lgbm.py index f66799b3..7d6bc04e 100644 --- a/examples/lgbm.py +++ b/examples/lgbm.py @@ -3,8 +3,8 @@ Example taken from https://mlfromscratch.com/gridsearch-keras-sklearn/#/ """ -from tune_sklearn import TuneSearchCV import lightgbm as lgb +from tune_sklearn import TuneSearchCV from sklearn.datasets import load_breast_cancer from sklearn.model_selection import train_test_split diff --git a/tune_sklearn/_trainable.py b/tune_sklearn/_trainable.py index 3aea4775..cc8429e2 100644 --- a/tune_sklearn/_trainable.py +++ b/tune_sklearn/_trainable.py @@ -12,13 +12,7 @@ import ray.cloudpickle as cpickle import warnings - -def try_import_xgboost(): - try: - import xgboost # ignore: F401 - return True - except ImportError: - return False +from tune_sklearn._detect_xgboost import is_xgboost_model class _Trainable(Trainable): @@ -37,13 +31,6 @@ def _can_partial_fit(self): def main_estimator(self): return self.estimator_list[0] - @property - def is_xgb(self): - if try_import_xgboost(): - from xgboost.sklearn import XGBModel - return isinstance(self.main_estimator, XGBModel) - return False - def setup(self, config): # forward-compatbility self._setup(config) @@ -90,7 +77,7 @@ def _setup(self, config): for i in range(n_splits): self.estimator_list[i].set_params(**self.estimator_config) - if self.is_xgb: + if is_xgboost_model(self.main_estimator): self.saved_models = [None for _ in range(n_splits)] else: self.main_estimator.set_params(**self.estimator_config) @@ -133,10 +120,11 @@ def _train(self): if self.is_xgb: self.estimator_list[i].fit( X_train, y_train, xgb_model=self.saved_models[i]) - self.saved_models[i] = self.estimator_list[i].get_booster() + self.saved_models[i] = self.estimator_list[ + i].get_booster() else: - self.estimator_list[i].partial_fit(X_train, y_train, - np.unique(self.y)) + self.estimator_list[i].partial_fit( + X_train, y_train, np.unique(self.y)) else: self.estimator_list[i].fit(X_train, y_train) diff --git a/tune_sklearn/tune_basesearch.py b/tune_sklearn/tune_basesearch.py index c2fed0aa..a27bd02a 100644 --- a/tune_sklearn/tune_basesearch.py +++ b/tune_sklearn/tune_basesearch.py @@ -19,8 +19,6 @@ from sklearn.base import is_classifier from sklearn.base import clone from sklearn.exceptions import NotFittedError -from lightgbm import LGBMModel -from xgboost.sklearn import XGBModel import ray from ray.tune.schedulers import ( PopulationBasedTraining, AsyncHyperBandScheduler, HyperBandScheduler, @@ -32,6 +30,8 @@ import multiprocessing import os +from tune_sklearn._detect_xgboost import is_xgboost_model + def resolve_early_stopping(early_stopping, max_iters): if isinstance(early_stopping, str): @@ -459,8 +459,7 @@ def _can_early_stop(self): and is_not_ensemble_subclass and is_not_tree_subclass) - is_gbm = isinstance(self.estimator, LGBMModel) or isinstance( - self.estimator, XGBModel) + is_gbm = is_xgboost_model(self.estimator) return can_partial_fit or can_warm_start or is_gbm From 0bb7bccedffd3f13d8495a1e96a695ddd8749b2c Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sat, 29 Aug 2020 18:06:35 -0700 Subject: [PATCH 43/47] remove-checkpoint-at-end --- tune_sklearn/tune_gridsearch.py | 2 -- tune_sklearn/tune_search.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/tune_sklearn/tune_gridsearch.py b/tune_sklearn/tune_gridsearch.py index b8d9003a..7e338a59 100644 --- a/tune_sklearn/tune_gridsearch.py +++ b/tune_sklearn/tune_gridsearch.py @@ -206,7 +206,6 @@ def _tune_run(self, config, resources_per_trial): stop={"training_iteration": self.max_iters}, config=config, fail_fast=True, - checkpoint_at_end=True, resources_per_trial=resources_per_trial, local_dir=os.path.expanduser(self.local_dir)) else: @@ -218,7 +217,6 @@ def _tune_run(self, config, resources_per_trial): stop={"training_iteration": self.max_iters}, config=config, fail_fast=True, - checkpoint_at_end=True, resources_per_trial=resources_per_trial, local_dir=os.path.expanduser(self.local_dir)) diff --git a/tune_sklearn/tune_search.py b/tune_sklearn/tune_search.py index 8284c67d..8849e45d 100644 --- a/tune_sklearn/tune_search.py +++ b/tune_sklearn/tune_search.py @@ -557,7 +557,6 @@ def _tune_run(self, config, resources_per_trial): num_samples=self.num_samples, config=config, fail_fast=True, - checkpoint_at_end=True, resources_per_trial=resources_per_trial, local_dir=os.path.expanduser(self.local_dir)) @@ -619,7 +618,6 @@ def _tune_run(self, config, resources_per_trial): num_samples=self.num_samples, config=config, fail_fast=True, - checkpoint_at_end=True, resources_per_trial=resources_per_trial, local_dir=os.path.expanduser(self.local_dir)) From 4ae730c06f9867ec33de7283f50e6fb85340b736 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sat, 29 Aug 2020 18:06:46 -0700 Subject: [PATCH 44/47] soft-dep --- tune_sklearn/_detect_xgboost.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 tune_sklearn/_detect_xgboost.py diff --git a/tune_sklearn/_detect_xgboost.py b/tune_sklearn/_detect_xgboost.py new file mode 100644 index 00000000..c27eca07 --- /dev/null +++ b/tune_sklearn/_detect_xgboost.py @@ -0,0 +1,12 @@ +def has_xgboost(): + try: + import xgboost # ignore: F401 + return True + except ImportError: + return False + +def is_xgboost_model(clf): + if not has_xgboost(): + return False + import xgboost # ignore: F401 + return isinstance(clf, XGBModel) From e18c12957df79d7afcc9af80fb6aee3283fca2ed Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sat, 29 Aug 2020 18:13:11 -0700 Subject: [PATCH 45/47] fix --- tune_sklearn/_trainable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tune_sklearn/_trainable.py b/tune_sklearn/_trainable.py index cc8429e2..6d6aaca8 100644 --- a/tune_sklearn/_trainable.py +++ b/tune_sklearn/_trainable.py @@ -117,7 +117,7 @@ def _train(self): test, train_indices=train) if self._can_partial_fit(): - if self.is_xgb: + if is_xgboost_model(self.main_estimator): self.estimator_list[i].fit( X_train, y_train, xgb_model=self.saved_models[i]) self.saved_models[i] = self.estimator_list[ From af169e686ac3d384f7a2e55a6c7723e5e36dffec Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sat, 29 Aug 2020 18:18:53 -0700 Subject: [PATCH 46/47] sklearn --- tune_sklearn/tune_search.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tune_sklearn/tune_search.py b/tune_sklearn/tune_search.py index 8849e45d..a5bf8be9 100644 --- a/tune_sklearn/tune_search.py +++ b/tune_sklearn/tune_search.py @@ -210,7 +210,7 @@ class TuneSearchCV(TuneBaseSearchCV): However computing the scores on the training set can be computationally expensive and is not strictly required to select the parameters that yield the best generalization performance. - local_dir (str): A string that defines where checkpoints will + local_dir (str): A string that defines where checkpoints and logs will be stored. Defaults to "~/ray_results" max_iters (int): Indicates the maximum number of epochs to run for each hyperparameter configuration sampled (specified by ``n_trials``). @@ -232,8 +232,8 @@ class TuneSearchCV(TuneBaseSearchCV): All types of search aside from Randomized search require parent libraries to be installed. use_gpu (bool): Indicates whether to use gpu for fitting. - Defaults to False. If True, training will use 1 gpu - for `resources_per_trial`. + Defaults to False. If True, training will start processes + with the proper CUDA VISIBLE DEVICE settings set. **search_kwargs (Any): Additional arguments to pass to the SearchAlgorithms (tune.suggest) objects. From a74b1fc023653d588d751949c412521f4b2374d3 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Sat, 29 Aug 2020 18:33:47 -0700 Subject: [PATCH 47/47] rename --- examples/{torch_nn.py => skorch_example.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/{torch_nn.py => skorch_example.py} (100%) diff --git a/examples/torch_nn.py b/examples/skorch_example.py similarity index 100% rename from examples/torch_nn.py rename to examples/skorch_example.py