---
title: "Tuning"
output:
    pdf_document:
        fig_height: 4
---

# Getting Parameters

```{r}
library(mlr)

lrn = makeLearner("classif.rpart")
getParamSet(lrn)
```

# Random Search

```{r}
ps = makeParamSet(
    makeIntegerParam("minsplit", lower = 1, upper = 100),
    makeIntegerParam("maxdepth", lower = 2, upper = 30)
)
ctrl = makeTuneControlRandom(maxit = 50)
rdesc = makeResampleDesc("CV", iters = 3) 
res = tuneParams(lrn, task = iris.task, resampling = rdesc, par.set = ps, control = ctrl)
res
```

## Tuning Results

```{r}
res$opt.path
```

```{r}
plotOptPath(res$opt.path)
```

```{r}
data = generateHyperParsEffectData(res)
plotHyperParsEffect(data, x = "iteration", y = "mmce.test.mean", plot.type = "line")
```

```{r}
plotHyperParsEffect(data, x = "minsplit", y = "mmce.test.mean", plot.type = "line")
```

```{r}
plotHyperParsEffect(data, x = "maxdepth", y = "mmce.test.mean", plot.type = "line")
```

## Using Results

```{r}
lrn = setHyperPars(makeLearner("classif.rpart"), par.vals = res$x)
lrn
```

## Wrapper and Nested Resampling

```{r}
inner = makeResampleDesc("Subsample", iters = 2)
lrn = makeTuneWrapper("classif.rpart", resampling = inner, par.set = ps, control = ctrl)

outer = makeResampleDesc("CV", iters = 3)
res = resample(lrn, iris.task, resampling = outer, extract = getTuneResult)
res
```

### Results

```{r}
opt.paths = getNestedTuneResultsOptPathDf(res)
opt.paths
```

```{r}
library(ggplot2)
ggplot(opt.paths, aes(x = minsplit, y = maxdepth, fill = mmce.test.mean)) + geom_tile() +
    facet_wrap(~ iter)
```

# Model-Based Tuning

## Numeric Parameters

```{r}
library(mlrMBO)

control = makeMBOControl()
control = setMBOControlTermination(control, iters = 10)
control = setMBOControlInfill(control, crit = makeMBOInfillCritEI())

ctrl = makeTuneControlMBO(mbo.control = control)

lrn = makeTuneWrapper("classif.rpart", resampling = inner, par.set = ps, control = ctrl)
res = resample(lrn, iris.task, resampling = outer, extract = getTuneResult)
res
```

## Combining Feature and Model Selection

```{r}
ps = makeParamSet(
    makeDiscreteParam("fw.method", values = c("anova.test", "variance")),
    makeNumericParam("fw.perc", lower = 0.1, upper = 1),
    makeDiscreteParam("selected.learner", values = c("classif.svm", "classif.naiveBayes")),
    makeNumericParam("classif.svm.cost", -15, 15, trafo = function(x) 2^x,
      require = quote(selected.learner == "classif.svm")),
    makeNumericParam("classif.svm.gamma", -15, 15, trafo = function(x) 2^x,
      requires = quote(classif.svm.kernel == "radial" & selected.learner == "classif.svm")),
    makeIntegerParam("classif.svm.degree", lower = 1, upper = 4,
      requires = quote(classif.svm.kernel == "polynomial" & selected.learner == "classif.svm")),
    makeDiscreteParam("classif.svm.kernel", values = c("radial", "polynomial", "linear"),
      require = quote(selected.learner == "classif.svm"))
)

ctrl = makeMBOControl()
ctrl = setMBOControlTermination(ctrl, iters = 10)
lrn = makeFilterWrapper(makeModelMultiplexer(list("classif.svm", "classif.naiveBayes")))
tune.ctrl = makeTuneControlMBO(mbo.control = ctrl)

lrn = makeTuneWrapper(lrn, resampling = inner, par.set = ps, control = tune.ctrl)
res = resample(lrn, iris.task, resampling = outer, extract = getTuneResult)
res
```

### Results

```{r}
opt.paths = getNestedTuneResultsOptPathDf(res)
opt.paths
```

```{r}
plotOptPath(res$extract[[1]]$opt.path)
```
```{r}
plotOptPath(res$extract[[2]]$opt.path)
```
```{r}
plotOptPath(res$extract[[3]]$opt.path)
```

```{r}
ggplot(opt.paths, aes(x = 1:nrow(opt.paths), y = cummin(mmce.test.mean))) +
    geom_line() + ylab("error") + xlab("iteration")
```

```{r}
best = getNestedTuneResultsX(res)
best
```
