from sklearn.model_selection import cross_val_score
from sklearn import datasets
from sklearn import svm
from sklearn.ensemble import RandomForestRegressor

import numpy as np
import random

iris = datasets.load_iris()

params = { 'C': np.logspace(-2, 10, 13),
               'gamma': np.logspace(-9, 3, 13) }
param_grid = [ { 'C': x, 'gamma': y } for x in params['C']
                                      for y in params['gamma'] ]
# [{'C': 0.01, 'gamma': 1e-09}, {'C': 0.01, 'gamma': 1e-08}...]

initial_samples = 3
evals = 10
random.seed(1)

def est_acc(pars):
    clf = svm.SVC(**pars)
    return np.median(cross_val_score(clf, iris.data, iris.target, cv = 10))

data = []
for pars in random.sample(param_grid, initial_samples):
    acc = est_acc(pars)
    data += [ list(pars.values()) + [ acc ] ]
# [[1.0, 0.1, 1.0],
#  [1000000000.0, 1e-07, 1.0],
#  [0. 1, 1e-06,0.9333333333333333]]

regr = RandomForestRegressor(random_state = 0)
for evals in range(0, evals):
    df = np.array(data)
    regr.fit(df[:,0:2], df[:,2])

    preds = regr.predict([ list(pars.values()) for pars in param_grid ])
    i = preds.argmax()

    acc = est_acc(param_grid[i])
    data += [ list(param_grid[i].values()) + [ acc ] ]
    print("{}: best predicted {} for {}, actual {}"
            .format(evals, round(preds[i], 2), param_grid[i], round(acc, 2)))

i = np.array(data)[:,2].argmax()
print("Best accuracy ({}) for parameters {}".format(data[i][2], data[i][0:2]))