Overview of the most important libraries for hyperparameter optimization - how to use them, what are the most important features, what are drawbacks, backed by performance review and recommendations.

1. Jakub Czakon

2. ●
●
○
○
●

3. scoredata
https://bbabenko.github.io/convolutional-learnings/

4. scoredata

5. ●
●
●
●

6. ●
●
●
●
●

7. ●
●
●
●
●

11. ●
○
○
○
●

12. SPACE = [skopt.space.Real(0.01, 0.5, name='learning_rate', prior='log-uniform'),
skopt.space.Integer(1, 30, name='max_depth'),
skopt.space.Integer(2, 100, name='num_leaves'),
skopt.space.Integer(10, 1000, name='min_data_in_leaf'),
skopt.space.Real(0.1, 1.0, name='feature_fraction', prior='uniform'),
skopt.space.Real(0.1, 1.0, name='subsample', prior='uniform'),
]

13. ●
●

14. def objective(**params):
return -1.0 * train_evaluate(X, y, **params)
@skopt.utils.use_named_args(SPACE)
def objective(**params):
return -1.0 * train_evaluate(X, y, **params)

15. ●
○
○
○
●

16. results = skopt.forest_minimize(objective, SPACE,
callback=[monitor],
n_calls=100,
n_random_starts=10,
base_estimator='ET',
acq_func='LCB',
xi=0.02,
kappa=1.96)

17. def monitor(res):
neptune.send_metric('run_score', res.func_vals[-1])

18. ●
●
○
○
●
○
○

19. ●
●
●

20. ●
○
○
○
●

21. skopt.plots.plot_convergence(results)

22. skopt.plots.plot_convergence(results_list)

23. skopt.plots.plot_evaluations(results)

24. skopt.plots.plot_evaluations(results)

25. skopt.plots.plot_evaluations(results)

26. skopt.plots.plot_objective(results)

27. ● cannot distribute it
● n_jobs
● 1 machine it is fast

28. ●
●
● skopt

30. ●
●
●
●
●

31. ●
●

34. def objective(trial):
params = OrderedDict([
('learning_rate',trial.suggest_loguniform('learning_rate', 0.01, 0.5)),
('max_depth',trial.suggest_int('max_depth', 1, 30)),
('num_leaves',trial.suggest_int('num_leaves', 2, 100)),
('min_data_in_leaf',trial.suggest_int('min_data_in_leaf', 10, 1000)),
('feature_fraction',trial.suggest_uniform('feature_fraction', 0.1, 1.0)),
('subsample',trial.suggest_uniform('subsample', 0.1, 1.0))])
score = -1.0 * train_evaluate(X, y, params)
return score

35. ●
○
○
○
●

36. def objective(trial):
classifier_name = trial.suggest_categorical('classifier', ['SVC', 'RandomForest'])
if classifier_name == 'SVC':
svc_c = trial.suggest_loguniform('svc_c', 1e-10, 1e10)
classifier_obj = sklearn.svm.SVC(C=svc_c)
else:
rf_max_depth = int(trial.suggest_loguniform('rf_max_depth', 2, 32))
classifier_obj = sklearn.ensemble.RandomForestClassifier(max_depth=rf_max_depth)
…

37. ●
●
●
●

38. study = optuna.create_study()
study.optimize(objective, n_trials=100])
results = study.trails_dataframe()

39. from optuna.integration import LightGBMPruningCallback
def objective(trial):
params = OrderedDict([
('max_depth',trial.suggest_int('max_depth', 1, 30)),
('num_leaves',trial.suggest_int('num_leaves', 2, 100))])
pruning_callback = LightGBMPruningCallback(trial, 'auc')
score = -1.0 * train_evaluate_with_pruning(X, y, params, pruning_callback)
return score
def train_evaluate_with_pruning(X, y, params, callback):
...
model = lgb.train(params, train_data, ... , callbacks = [pruning_callback])
return model.best_score['valid']['auc']

40. def monitor(score):
neptune.send_metric('run_score', score)
def objective(trial):
params = OrderedDict([
('max_depth',trial.suggest_int('max_depth', 1, 30)),
('num_leaves',trial.suggest_int('num_leaves', 2, 100))])
score = -1.0 * train_evaluate(X, y, params)
monitor(score)
return score

41. ●
●
●

43. ● Can be distributed
● Has pruning

44. study.optimize(objective, n_trials=100, n_jobs=5)

45. …
study = optuna.Study(study_name='distributed-search', storage='sqlite:///example.db')
study.optimize(objective, n_trials=100)
...
$ optuna create-study --study-name "distributed-search" --storage "sqlite:///example.db"
$ python optuna_search.py
$ python optuna_search.py
terminal 1
terminal 2
terminal 3
optuna_search.py

46. ●
●
● optuna

48. ●
●
●
●
●

49. ●
●
●

50. ●
●
●
●
●
●

53. ●
●
○
○
●

56. ●
○
○
●

57. import hpbandster.core.nameserver as hpns
NS = hpns.NameServer(run_id=RUN_ID, host=HOST, port=PORT, working_directory=WORKING_DIRECTORY)
ns_host, ns_port = NS.start()

58. from hpbandster.core.worker import Worker
class TrainEvalWorker(Worker):
...
def compute(self, config, budget, working_directory, *args, **kwargs):
loss = -1.0 * train_evaluate(self.X, self.y, params)
return ({'loss': loss,
'info': { 'auxiliary_stuff': 'worked'
}
})

59. ●
○
○
○
●

60. class TrainEvalWorker(Worker):
...
@staticmethod
def get_configspace():
cs = CS.ConfigurationSpace()
learning_rate = CSH.UniformFloatHyperparameter('learning_rate',
lower=0.01, upper=0.5, default_value=0.01, log=True)
subsample = CSH.UniformFloatHyperparameter('subsample',
lower=0.1, upper=1.0, default_value=0.5, log=False)
cs.add_hyperparameters([learning_rate, subsample])
return cs

61. worker = TrainEvalWorker(run_id=RUN_ID, nameserver=ns_host, nameserver_port=ns_port)
worker.run(background=True)

62. from hpbandster.optimizers import BOHB
optim = BOHB(configspace = worker.get_configspace(),
run_id = RUN_ID,
nameserver=ns_host,
nameserver_port=ns_port,
result_logger=NeptuneLogger(),
eta=3, min_budget=0.1, max_budget=1,
num_samples=64, top_n_percent=15,
min_bandwidth=1e-3, bandwidth_factor=3)
study = optim.run(n_iterations=100)

63. class NeptuneLogger:
def new_config(self, *args, **kwargs):
pass
def __call__(self, job):
neptune.send_metric('run_score', job.result['loss'])
neptune.send_text('run_parameters', str(job.kwargs['config']))

64. ●
●
●

65. ●
○
○
○
○
○
●

66. all_runs = results.get_all_runs()
hpvis.losses_over_time(all_runs);

67. all_runs = results.get_all_runs()
hpvis.concurrent_runs_over_time(all_runs);

68. all_runs = results.get_all_runs()
hpvis.finished_runs_over_time(all_runs);

69. hpvis.correlation_across_budgets(results);

71. all_runs = results.get_all_runs()
id2conf = results.get_id2config_mapping()
hpvis.performance_histogram_model_vs_random(all_runs, id2conf);

73. ● Can be distributed

74. workers=[]
for i in range(N_WORKERS):
w = TrainEvalWorker(run_id=RUN_ID, id=isleep_interval = 0.5,
nameserver=ns_host, nameserver_port=ns_port)
w.run(background=True)
workers.append(w)
optim = BOHB(configspace = TrainEvalWorker.get_configspace(),
run_id = RUN_ID,
nameserver=ns_host,
nameserver_port=ns_port)
study = optim.run(n_iterations=100, min_n_workers=N_WORKERS)

75. workers=[]
for i in range(N_WORKERS):
w = TrainEvalWorker(run_id=RUN_ID, id=isleep_interval = 0.5,
nameserver=ns_host, nameserver_port=ns_port)
w.run(background=False)
exit(0)
optim = BOHB(configspace = TrainEvalWorker.get_configspace(),
run_id = RUN_ID,
nameserver=ns_host,
nameserver_port=ns_port)
study = optim.run(n_iterations=100, min_n_workers=N_WORKERS)

77. ●
●
● hpbandster

79. ●
●
●
●

80. ●
●

82. Scikit-Optimize Optuna HpBandSter Hyperopt
API/ease of use Great Great Difficult Good
Documentation Great Great Ok(ish) Bad
Speed/Parallelization Fast if
sequential/None
Great Good Ok
Visualizations Amazing None Very lib specific Some
*Experimental results 0.8566 (100) 0.8419 (100)
0.8597 (10000)
0.8629 (100) 0.8420 (100)

84. import neptunecontrib.hpo.utils as hpo_utils
results = hpo_utils.optuna2skopt(study)

85. ● Scikit-Optimize
● HpBandSter
● Optuna

86. jakub@neptune.ml
@NeptuneML
https://medium.com/@jakub.czakon
https://twitter.com/neptuneml