---
title: "SPOTMiscVignette"
output: rmarkdown::html_vignette
vignette: >
  %\VignetteIndexEntry{SPOTMiscVignette}
  %\VignetteEngine{knitr::rmarkdown}
  %\VignetteEncoding{UTF-8}
---

```{r knitrSetup, include = FALSE}
knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>"
)
```

# R Installation of Missing Packages

```{r, eval = FALSE}
install.packages(c( 
"benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets"))
```


# Python Installation


```{r, eval = FALSE}
cd <project-dir>
```

Create a new virtual environment in a folder called python within the project directory using the following command:

```{r, eval = FALSE}
virtualenv python
```

The virtualenv can be activated using the following command in a terminal:

```{r, eval = FALSE}
source python/bin/activate
```


To verify that the correct version of python was activated the following command can be executed in a terminal:
```{r, eval = FALSE}
which python
```

## Install Python packages in the Environment

Python packages such as tensorflow-datasets, numpy, pandas, matplotlib, and other packages can be installed in the  Python virtualenv by using
pip install:
```{r, eval = FALSE}
pip install numpy pandas matplotlib tensorflow tensorflow-datasets
```

```{r, eval=FALSE}
/Users/bartz/workspace/SPOT/python/bin/python -m pip install --upgrade pip
```


## Install and Configure reticulate to use the Correct Python Version

Install the  reticulate package using the following command in the R console (e.g., from within RStudio):
```{r, eval = FALSE}
install.packages("reticulate")
```

To configure reticulate to point to the Python executable in the virtualenv python from above, 
create a file in the project directory called .Rprofile with the following contents:


```{r, eval = FALSE}
Sys.setenv(RETICULATE_PYTHON = "python/bin/python")
```

R (or RStudio) must be restarted for the setting to take effect.
To check that reticulate is configured for the correct version of Python the following command can be used in the R (or RStudio) console:


```{r, eval = FALSE}
reticulate::py_config()
```

## Installing Keras

To get started with Keras, the Keras R package, the core Keras library, and a backend tensor engine (such as TF) must be installed. 
This can be done as follows from within R (or RStudio):

```{r, eval = FALSE}
install.packages("tensorflow")
install.packages("keras")
library(keras)
install_keras()
```


This will provide you with a default CPU-based installation of Keras and TF.
To install a GPU-based version of the TF backend engine, the corresponding command reads as follows:
```{r, eval = FALSE}
install_keras(tensorflow = "gpu")
```


```{r SPOT, include = FALSE, eval = FALSE}
## install.packages("devtools")
## devtools::install_github("r-lib/devtools")
url <- "http://owos.gm.fh-koeln.de:8055/bartz/spot.git"
devtools::install_git(url = url)
```

* Package version SPOT should be larger than `2.3.0`.

```{r, setup, eval=FALSE}
library("SPOT")
packageVersion("SPOT")
```


### With noise: (this takes some time)

```{r, spotNoise, eval = FALSE}
# noisy objective
res1 <- spot(,function(x)funSphere(x)+rnorm(nrow(x)),c(-2,-3),c(1,2),
 		control=list(funEvals=40,noise=TRUE)) 
# noise with replicated evaluations
res2 <- spot(,function(x)funSphere(x)+rnorm(nrow(x)),c(-2,-3),c(1,2),
 		control=list(funEvals=40,noise=TRUE,replicates=2,
 		designControl=list(replicates=2))) 
# and with OCBA
res3 <- spot(,function(x)funSphere(x)+rnorm(nrow(x)),c(-2,-3),c(1,2),
 		control=list(funEvals=40,noise=TRUE,replicates=2,OCBA=TRUE,OCBABudget=1,
 		designControl=list(replicates=2))) 
# Check results with non-noisy function:
funSphere(res1$xbest)
funSphere(res2$xbest)
funSphere(res3$xbest)
```


### Handling factor variables

Note: factors should be coded as integer values, i.e., 1,2,...,n
First, we create a test function with a factor variable:

```{r braninFactor, eval=FALSE}
braninFunctionFactor <- function (x) {
   y <- (x[2]  - 5.1/(4 * pi^2) * (x[1] ^2) + 5/pi * x[1]  - 6)^2 +
     10 * (1 - 1/(8 * pi)) * cos(x[1] ) + 10
   if(x[3]==1)
     y <- y +1
   else if(x[3]==2)
     y <- y -1
   return(y)
}
```

Vectorize the test function.

```{r vecObj, eval = FALSE}
objFun <- function(x){apply(x,1,braninFunctionFactor)}
```

Run `spot`.

```{r, spotFac, eval = FALSE}
set.seed(1)
res <- spot(fun=objFun,lower=c(-5,0,1),upper=c(10,15,3),
            control=list(model=buildKriging,
                         types= c("numeric","numeric","factor"),
                         optimizer=optimLHD))
 res$xbest
 res$ybest
```

### Run SPOT with logging

```{r splotLag, eval = FALSE}
## run spot without log
res <- spot(fun = funSphere,
            lower=c(0,0),
            upper=c(100,100)
)
## run spot with log
funSphereLog <- function(x){
  cbind(funSphere(x),x)
}
res2 <- spot(fun = funSphereLog,
            lower=c(0,0),
            upper=c(100,100)
)
res$logInfo
res2$logInfo
```

## Keras/Tensorflow

### First experiments (from the arxiv paper summer 2021)

#### exp01
```{r, eval=FALSE}
library("SPOTMisc")
library("SPOT")
kerasConf <- getKerasConf()
lower <- c(1e-6, 1e-6, 16,0.6, 1e-9, 10, 6,0.4,0.99,1,1e-8)
upper <- c(0.5, 0.5, 512, 1.5, 1e-2, 50, 10,0.999,0.999,10,6e-8)
types <- c("numeric",  "numeric",  "integer",  "numeric",  "numeric",
           "integer",  "integer",  "numeric",  "numeric",  "integer",
           "numeric")


kerasConf$verbose <- 0
res <- spot(x = NULL,
            fun = funKerasMnist,
            lower = lower,
            upper = upper,
            control = list(funEvals=100,
                           noise = TRUE,
                           types = types,
                           plots = TRUE,
                           progress = TRUE,
                           seedFun = 1,
                           seedSPOT = 1),
            kConf = kerasConf)
res01 <- res
# save(res01, file="exp01.RData")
```

#### exp02

```{r, eval=FALSE}
library("SPOTMisc")
library("SPOT")
kerasConf <- getKerasConf()
lower <- c(1e-6, 1e-6, 16,0.6, 1e-9, 10, 6,0.4,0.99,1,1e-8)
upper <- c(0.5, 0.5, 512, 1.5, 1e-2, 50, 10,0.999,0.999,10,6e-8)
types <- c("numeric",  "numeric",  "integer",  "numeric",  "numeric",
           "integer",  "integer",  "numeric",  "numeric",  "integer",
           "numeric")


kerasConf$verbose <- 0
Ninit <- 3 * length(lower)
Rinit <- 3
replicates <- 3

res <- spot(x = NULL,
            fun = funKerasMnist,
            lower = lower,
            upper = upper,
            control = list(funEvals=20*length(lower),
                           noise = TRUE,
                           types = types,
                           replicates=replicates,
                           designControl = list(replicates = Rinit,
                                                               size = Ninit),
                           model = buildKriging,
                           optimizer = optimLBFGSB,
                           modelControl=list(target="ei"),
                           plots = TRUE,
                           progress = TRUE,
                           seedFun = 1,
                           seedSPOT = 1),
            kConf = kerasConf)
res02 <- res
#save(res02, file="exp02.RData")
```



### run002

```{r run002, eval=FALSE}
#install.packages("~/workspace/SPOTMisc_1.2.26.tar.gz", repos=NULL, type="source")
#install.packages("~/workspace/SPOT_2.6.2.tar.gz", repos=NULL, type="source")
library("keras")
library("SPOT")
library("SPOTMisc")
packageVersion("SPOT")
packageVersion("SPOTMisc")
set.seed(123)

dat <- genCatsDogsData()
kerasConf <- getKerasConf()
kerasConf$trainData <- dat$trainData
kerasConf$validationData <- dat$validationData
kerasConf$testData <- dat$testData

lower <- c(1e-6, 1e-6, 16,0.6, 1e-9, 10, 6,0.4,0.99,1,1e-8)
upper <- c(0.5, 0.5, 512, 1.5, 1e-2, 50, 10,0.999,0.999,10,6e-8)
types <- c("numeric",  "numeric",  "integer",  "numeric",  "numeric",
           "integer",  "integer",  "numeric",  "numeric",  "integer",
           "numeric")
### First example: simple function call:
x <- matrix(lower, 1,)
funKerasTransferLearning(x, kConf = kerasConf)
kerasConf$verbose <- 1
res <- spot(x = NULL,
            fun = funKerasTransferLearning,
            lower = lower,
            upper = upper,
            control = list(funEvals=200,
                           noise = TRUE,
                           types = types,
                           plots = TRUE,
                           progress = TRUE,
                           seedFun = 1,
                           seedSPOT = 1),
            kConf = kerasConf)
# save(res, file = "run002.RData")
```



## keras/TF Experiments 2022

### kerasMnist02

```{r, eval = FALSE}
install.packages(c( 
"benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets"))
install.packages(
  "~/workspace/spotmisc/packages/SPOT_2.7.0.tar.gz",
  repos = NULL,
  type = "source"
)
install.packages(
  "~/workspace/spotmisc/packages/SPOTMisc_1.6.0.tar.gz",
  repos = NULL,
  type = "source"
)
library("SPOTMisc")
packageVersion("SPOTMisc")
library("SPOT")
packageVersion("SPOT")

mnist <- getMnistData()

# "dropout" =  x[1]: 0.1--0.5
# "dropoutfact" =  x[2]: 0.85--0.95
# "units" = x[3]: 7-9 (128--512)
# "unitsfact" = x[4]:  0.5--1.5
# "learning_rate" =  x[5]: 1e-5--1e-2
# "epochs" = x[6]: 3-6 (10--50)
# "batchsize" = x[7]: 6-9 (64--512)
# "beta_1" =  x[8]: 0.9--0.999
# "beta_2" =  x[9]: 0.99--0.9999
# "layers" =  x[10]: 1--5
# "epsilon" = x[11]: 1e-8--1e-8
lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3,  6,  0.9,  0.99,    1, 1e-8)
upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6)
types <-  rep("numeric", length(lower))
types[c(3, 6, 7)] <- "integer"
f2 <- function(x) {
  2 ^ x
}
transformFun <- rep("identity", length(lower))
transformFun[c(3, 6, 7)] <- "f2"
kerasConf <- getKerasConf()
kerasConf$verbose <- 0
kerasConf$resDummy <- FALSE
Ninit <- 2 * length(lower)
Rinit <- 2
replicates <- 2

res <- spot(
  x = NULL,
  fun = funKerasMnist,
  lower = lower,
  upper = upper,
  control = list(
    funEvals = 20 * length(lower),
    noise = TRUE,
    types = types,
    transformFun = transformFun,
    replicates = replicates,
    designControl = list(replicates = Rinit,
                         size = Ninit),
    model = buildKriging,
    optimizer = optimLBFGSB,
    modelControl = list(target = "ei"),
    plots = FALSE,
    progress = TRUE,
    seedFun = 1,
    seedSPOT = 1
  ),
  kConf = kerasConf,
  data = mnist
)
save(res, file = paste0("resKerasMnist02", as.numeric(Sys.time()), ".RData"))
##
## Evaluation of the default hyperparameter setting (baseline)
library("SPOTMisc")
packageVersion("SPOTMisc")
##
source("~/workspace/spotmisc/R/funKerasMnist.R")
library("SPOT")
packageVersion("SPOT")
source("~/workspace/SPOT/R/spot.R")
source("~/workspace/SPOT/R/objectiveFunctionEvaluation.R")
mnist <- getMnistData()
# "dropout" =  x[1],
# "dropoutfact" =  x[2],
# "units" = x[3],
# "unitsfact" = x[4],
# "learning_rate" =  x[5],
# "epochs" = x[6],
# "batchsize" = x[7],
# "beta_1" =  x[8],
# "beta_2" =  x[9],
# "layers" =  x[10],
# "epsilon" = x[11]
lower <- c(0.4, 0.9, 8, 0.5, 0.001, 4, 3, 0.9, 0.999, 2, 1e-07)
upper <- lower
types <-  rep("numeric", length(lower))
types[c(3, 6, 7)] <- "integer"
f2 <- function(x) {
  2 ^ x
}
transformFun <- rep("identity", length(lower))
transformFun[c(3, 6, 7)] <- "f2"
#
kerasConf <- getKerasConf()
kerasConf$resDummy <- FALSE
kerasConf$verbose <- 0
kerasConf$returnValue <- "validationLoss"
kerasConf$naDummy <- 1234567890
kerasConf$clearSession <- TRUE
### Example: spot call to evaluate the default design
n <- 10
res <- spot(
  x = NULL,
  fun = funKerasMnist,
  lower = lower,
  upper = upper,
  control = list(
    funEvals = n,
    transformFun = transformFun,
    replicateResults = TRUE,
    types = types,
    seedFun = 1,
    seedSPOT = 1,
    verbosity = 0,
    designControl=list(size=n)
  ),
  kConf = kerasConf,
  data = mnist
)
save(res, file = paste0("resKerasMnist02Default", as.numeric(Sys.time()), ".RData"))
```



### KerasMnist03: same as 02, but training loss passed to optimizer

```{r resKerasMnist03, eval=FALSE}
install.packages(c( 
"benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets"))
install.packages(
  "~/workspace/spotmisc/packages/SPOT_2.7.0.tar.gz",
  repos = NULL,
  type = "source"
)
install.packages(
  "~/workspace/spotmisc/packages/SPOTMisc_1.6.0.tar.gz",
  repos = NULL,
  type = "source"
)
library("SPOTMisc")
packageVersion("SPOTMisc")
library("SPOT")
packageVersion("SPOT")

mnist <- getMnistData()

# "dropout" =  x[1]: 0.1--0.5
# "dropoutfact" =  x[2]: 0.85--0.95
# "units" = x[3]: 7-9 (128--512)
# "unitsfact" = x[4]:  0.5--1.5
# "learning_rate" =  x[5]: 1e-5--1e-2
# "epochs" = x[6]: 3-6 (10--50)
# "batchsize" = x[7]: 6-9 (64--512)
# "beta_1" =  x[8]: 0.9--0.999
# "beta_2" =  x[9]: 0.99--0.9999
# "layers" =  x[10]: 1--5
# "epsilon" = x[11]: 1e-8--1e-8
lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3,  6,  0.9,  0.99,    1, 1e-8)
upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6)
types <-  rep("numeric", length(lower))
types[c(3, 6, 7)] <- "integer"
f2 <- function(x) {
  2 ^ x
}
transformFun <- rep("identity", length(lower))
transformFun[c(3, 6, 7)] <- "f2"
kerasConf <- getKerasConf()
kerasConf$verbose <- 0
kerasConf$resDummy <- FALSE
kerasConf$returnValue <- "trainingLoss"
Ninit <- 2 * length(lower)
Rinit <- 2
replicates <- 2

res <- spot(
  x = NULL,
  fun = funKerasMnist,
  lower = lower,
  upper = upper,
  control = list(
    funEvals = 20 * length(lower),
    noise = TRUE,
    types = types,
    transformFun = transformFun,
    replicates = replicates,
    designControl = list(replicates = Rinit,
                         size = Ninit),
    model = buildKriging,
    optimizer = optimLBFGSB,
    modelControl = list(target = "ei"),
    plots = FALSE,
    progress = TRUE,
    seedFun = 1,
    seedSPOT = 1
  ),
  kConf = kerasConf,
  data = mnist
)
save(res, file = paste0("resKerasMnist03", as.numeric(Sys.time()),".RData"))
```


### KerasMnist07: same as 02, but test loss passed to optimizer

```{r resKerasMnist07, eval=FALSE}
install.packages(c( 
"benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets"))
install.packages(
  "~/workspace/spotmisc/packages/SPOT_2.7.0.tar.gz",
  repos = NULL,
  type = "source"
)
install.packages(
  "~/workspace/spotmisc/packages/SPOTMisc_1.6.0.tar.gz",
  repos = NULL,
  type = "source"
)
library("SPOTMisc")
packageVersion("SPOTMisc")
library("SPOT")
packageVersion("SPOT")

mnist <- getMnistData()

# "dropout" =  x[1]: 0.1--0.5
# "dropoutfact" =  x[2]: 0.85--0.95
# "units" = x[3]: 7-9 (128--512)
# "unitsfact" = x[4]:  0.5--1.5
# "learning_rate" =  x[5]: 1e-5--1e-2
# "epochs" = x[6]: 3-6 (10--50)
# "batchsize" = x[7]: 6-9 (64--512)
# "beta_1" =  x[8]: 0.9--0.999
# "beta_2" =  x[9]: 0.99--0.9999
# "layers" =  x[10]: 1--5
# "epsilon" = x[11]: 1e-8--1e-8
lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3,  6,  0.9,  0.99,    1, 1e-8)
upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6)
types <-  rep("numeric", length(lower))
types[c(3, 6, 7)] <- "integer"
f2 <- function(x) {
  2 ^ x
}
transformFun <- rep("identity", length(lower))
transformFun[c(3, 6, 7)] <- "f2"
kerasConf <- getKerasConf()
kerasConf$verbose <- 0
kerasConf$resDummy <- FALSE
kerasConf$returnValue <- "testLoss"
Ninit <- 2 * length(lower)
Rinit <- 2
replicates <- 2

res <- spot(
  x = NULL,
  fun = funKerasMnist,
  lower = lower,
  upper = upper,
  control = list(
    funEvals = 20 * length(lower),
    noise = TRUE,
    types = types,
    transformFun = transformFun,
    replicates = replicates,
    designControl = list(replicates = Rinit,
                         size = Ninit),
    model = buildKriging,
    optimizer = optimLBFGSB,
    modelControl = list(target = "ei"),
    plots = FALSE,
    progress = TRUE,
    seedFun = 1,
    seedSPOT = 1
  ),
  kConf = kerasConf,
  data = mnist
)
save(res, file = paste0("resKerasMnist07", as.numeric(Sys.time()),".RData"))
```

### KerasMnist08: same as 02, but negative validation accuracy passed to optimizer

```{r resKerasMnist08, eval=FALSE}
install.packages(c( 
"benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets"))
install.packages(
  "~/workspace/spotmisc/packages/SPOT_2.7.0.tar.gz",
  repos = NULL,
  type = "source"
)
install.packages(
  "~/workspace/spotmisc/packages/SPOTMisc_1.6.0.tar.gz",
  repos = NULL,
  type = "source"
)
library("SPOTMisc")
packageVersion("SPOTMisc")
library("SPOT")
packageVersion("SPOT")

mnist <- getMnistData()

# "dropout" =  x[1]: 0.1--0.5
# "dropoutfact" =  x[2]: 0.85--0.95
# "units" = x[3]: 7-9 (128--512)
# "unitsfact" = x[4]:  0.5--1.5
# "learning_rate" =  x[5]: 1e-5--1e-2
# "epochs" = x[6]: 3-6 (10--50)
# "batchsize" = x[7]: 6-9 (64--512)
# "beta_1" =  x[8]: 0.9--0.999
# "beta_2" =  x[9]: 0.99--0.9999
# "layers" =  x[10]: 1--5
# "epsilon" = x[11]: 1e-8--1e-8
lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3,  6,  0.9,  0.99,    1, 1e-8)
upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6)
types <-  rep("numeric", length(lower))
types[c(3, 6, 7)] <- "integer"
f2 <- function(x) {
  2 ^ x
}
transformFun <- rep("identity", length(lower))
transformFun[c(3, 6, 7)] <- "f2"
kerasConf <- getKerasConf()
kerasConf$verbose <- 0
kerasConf$resDummy <- FALSE
kerasConf$returnValue <- "negValidationAccuracy"
Ninit <- 2 * length(lower)
Rinit <- 2
replicates <- 2

res <- spot(
  x = NULL,
  fun = funKerasMnist,
  lower = lower,
  upper = upper,
  control = list(
    funEvals = 20 * length(lower),
    noise = TRUE,
    types = types,
    transformFun = transformFun,
    replicates = replicates,
    designControl = list(replicates = Rinit,
                         size = Ninit),
    model = buildKriging,
    optimizer = optimLBFGSB,
    modelControl = list(target = "ei"),
    plots = FALSE,
    progress = TRUE,
    seedFun = 1,
    seedSPOT = 1
  ),
  kConf = kerasConf,
  data = mnist
)
save(res, file = paste0("resKerasMnist08", as.numeric(Sys.time()),".RData"))
```

### KerasMnist09: same as 02, but negative training accuracy passed to optimizer

```{r resKerasMnist09, eval=FALSE}
install.packages(c( 
"benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets"))
install.packages(
  "~/workspace/spotmisc/packages/SPOT_2.7.0.tar.gz",
  repos = NULL,
  type = "source"
)
install.packages(
  "~/workspace/spotmisc/packages/SPOTMisc_1.6.0.tar.gz",
  repos = NULL,
  type = "source"
)
library("SPOTMisc")
packageVersion("SPOTMisc")
library("SPOT")
packageVersion("SPOT")

mnist <- getMnistData()

# "dropout" =  x[1]: 0.1--0.5
# "dropoutfact" =  x[2]: 0.85--0.95
# "units" = x[3]: 7-9 (128--512)
# "unitsfact" = x[4]:  0.5--1.5
# "learning_rate" =  x[5]: 1e-5--1e-2
# "epochs" = x[6]: 3-6 (10--50)
# "batchsize" = x[7]: 6-9 (64--512)
# "beta_1" =  x[8]: 0.9--0.999
# "beta_2" =  x[9]: 0.99--0.9999
# "layers" =  x[10]: 1--5
# "epsilon" = x[11]: 1e-8--1e-8
lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3,  6,  0.9,  0.99,    1, 1e-8)
upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6)
types <-  rep("numeric", length(lower))
types[c(3, 6, 7)] <- "integer"
f2 <- function(x) {
  2 ^ x
}
transformFun <- rep("identity", length(lower))
transformFun[c(3, 6, 7)] <- "f2"
kerasConf <- getKerasConf()
kerasConf$verbose <- 0
kerasConf$resDummy <- FALSE
kerasConf$returnValue <- "negTrainingAccuracy"
Ninit <- 2 * length(lower)
Rinit <- 2
replicates <- 2

res <- spot(
  x = NULL,
  fun = funKerasMnist,
  lower = lower,
  upper = upper,
  control = list(
    funEvals = 20 * length(lower),
    noise = TRUE,
    types = types,
    transformFun = transformFun,
    replicates = replicates,
    designControl = list(replicates = Rinit,
                         size = Ninit),
    model = buildKriging,
    optimizer = optimLBFGSB,
    modelControl = list(target = "ei"),
    plots = FALSE,
    progress = TRUE,
    seedFun = 1,
    seedSPOT = 1
  ),
  kConf = kerasConf,
  data = mnist
)
save(res, file = paste0("resKerasMnist09", as.numeric(Sys.time()),".RData"))
```

### KerasMnist10: same as 02, but negative test accuracy passed to optimizer

```{r resKerasMnist10, eval=FALSE}
install.packages(c( 
"benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets"))
install.packages(
  "~/workspace/spotmisc/packages/SPOT_2.7.0.tar.gz",
  repos = NULL,
  type = "source"
)
install.packages(
  "~/workspace/spotmisc/packages/SPOTMisc_1.6.0.tar.gz",
  repos = NULL,
  type = "source"
)
library("SPOTMisc")
packageVersion("SPOTMisc")
library("SPOT")
packageVersion("SPOT")

mnist <- getMnistData()

# "dropout" =  x[1]: 0.1--0.5
# "dropoutfact" =  x[2]: 0.85--0.95
# "units" = x[3]: 7-9 (128--512)
# "unitsfact" = x[4]:  0.5--1.5
# "learning_rate" =  x[5]: 1e-5--1e-2
# "epochs" = x[6]: 3-6 (10--50)
# "batchsize" = x[7]: 6-9 (64--512)
# "beta_1" =  x[8]: 0.9--0.999
# "beta_2" =  x[9]: 0.99--0.9999
# "layers" =  x[10]: 1--5
# "epsilon" = x[11]: 1e-8--1e-8
lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3,  6,  0.9,  0.99,    1, 1e-8)
upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6)
types <-  rep("numeric", length(lower))
types[c(3, 6, 7)] <- "integer"
f2 <- function(x) {
  2 ^ x
}
transformFun <- rep("identity", length(lower))
transformFun[c(3, 6, 7)] <- "f2"
kerasConf <- getKerasConf()
kerasConf$verbose <- 0
kerasConf$resDummy <- FALSE
kerasConf$returnValue <- "negTestAccuracy"
Ninit <- 2 * length(lower)
Rinit <- 2
replicates <- 2

res <- spot(
  x = NULL,
  fun = funKerasMnist,
  lower = lower,
  upper = upper,
  control = list(
    funEvals = 20 * length(lower),
    noise = TRUE,
    types = types,
    transformFun = transformFun,
    replicates = replicates,
    designControl = list(replicates = Rinit,
                         size = Ninit),
    model = buildKriging,
    optimizer = optimLBFGSB,
    modelControl = list(target = "ei"),
    plots = FALSE,
    progress = TRUE,
    seedFun = 1,
    seedSPOT = 1
  ),
  kConf = kerasConf,
  data = mnist
)
save(res, file = paste0("resKerasMnist10", as.numeric(Sys.time()),".RData"))
```




## Transfer Learning

### resKerasTransferLearning04

```{r, eval=FALSE}
library("SPOTMisc")
library("SPOT")

# Hyperparameters:
# "dropout" =  x[1],
# "learning_rate" =  x[2],
# "epochs" = x[3],
# "beta_1" =  x[4],
# "beta_2" =  x[5],
# "epsilon" = x[6],
# "optimizer" = x[7]

lower <- c(1e-6, 1e-6, 2, 0.8, 0.8, 1e-9, 1)
upper <- c(0.2, 1e-2, 5, 0.99, 0.9999, 1e-3, 2)
types <- c("numeric",  "numeric",  "integer",  "numeric",  "numeric",
                      "numeric",  "factor")

kerasConf <- getKerasConf()
kerasConf$verbose <- 1
kerasConf$resDummy <- FALSE
kerasConf$naDummy <- 1234567890
res <- spot(x = NULL,
            fun = funKerasTransferLearning,
            lower = lower,
            upper = upper,
            control = list(funEvals=50,
                           model=buildKriging,
                           noise = TRUE,
                           types = types,
                           optimizer=optimDE,
                           plots = FALSE,
                           progress = TRUE,
                           seedFun = 1,
                           seedSPOT = 1,
                           verbosity = 1), 
                           kConf = kerasConf
)
save(res, file = paste0("resKerasTransferLearning04", as.numeric(Sys.time()),".RData"))
```


## resKerasTransferLearning05

```{r, eval=FALSE}
library("SPOTMisc")
library("SPOT")
kerasConf <- getKerasConf()

# Hyperparameters:
# "dropout" =  x[1],
# "learning_rate" =  x[2],
# "epochs" = x[3],
# "beta_1" =  x[4],
# "beta_2" =  x[5],
# "epsilon" = x[6],
# "optimizer" = x[7]

lower <- c(1e-6, 1e-6, 2, 0.8, 0.8, 1e-9, 1)
upper <- c(0.2, 1e-2, 5, 0.99, 0.9999, 1e-3, 2)
types <- c("numeric",  "numeric",  "integer",  "numeric",  "numeric",
           "numeric",  "factor")

### Example: spot call with extended verbosity and negative val accuracy:
kerasConf$verbose <- 0
kerasConf$returnValue <- "negValidationAccuracy"
res <- spot(x = NULL,
            fun = funKerasTransferLearning,
            lower = lower,
            upper = upper,
            control = list(funEvals=50,
                           model=buildKriging,
                           noise = TRUE,
                           types = types,
                           optimizer=optimDE,
                           plots = FALSE,
                           progress = TRUE,
                           seedFun = 1,
                           seedSPOT = 1,
                           verbosity = 1), 
                           kConf = kerasConf
)
save(res, file = paste0("resKerasTransferLearning05", as.numeric(Sys.time()),".RData"))
```


## resKerasTransferLearning06

* Evaluation of the default hyperparameter setting (baseline)

```{r, eval=FALSE}
library("SPOTMisc")
library("SPOT")
kerasConf <- getKerasConf()

# Hyperparameters:
# "dropout" =  x[1],
# "learning_rate" =  x[2],
# "epochs" = x[3],
# "beta_1" =  x[4],
# "beta_2" =  x[5],
# "epsilon" = x[6],
# "optimizer" = x[7]
replicates <- 1
lower <- c(0.2, 1e-5, 5, 0.9, 0.999, 1e-7, 1)
upper <- c(0.2, 1e-5, 5, 0.9, 0.999, 1e-7, 1)
types <- c("numeric",  "numeric",  "integer",  "numeric",  "numeric",
           "numeric",  "factor")

### Example: spot call to evaluate the default design
kerasConf$verbose <- 0
kerasConf$returnValue <- "validationLoss"
kerasConf$naDummy <- 1234567890
res <- spot(x = NULL,
            fun = funKerasTransferLearning,
            lower = lower,
            upper = upper,
            control = list(funEvals=10,
                           model=buildKriging,
                           noise = TRUE,
                           types = types,
                           optimizer = optimDE,
                           plots = FALSE,
                           progress = TRUE,
                           designControl=list(replicates=replicates),
                           seedFun = 1,
                           seedSPOT = 1,
                           verbosity = 1), 
                           kConf = kerasConf
)
save(res, file = paste0("resKerasTransferLearning06", as.numeric(Sys.time()),".RData"))
```

### kerasMnist11:  02 with  Multi Start

```{r, eval = FALSE}
install.packages(c( 
"benchmarkme", "callr", "emoa", "ggsci", "jsonlite", "keras", "magrittr", "mlr", "plotly", "RColorBrewer", "reticulate", "rpart.plot", "sensitivity", "smoof", "tensorflow", "tfdatasets"))
install.packages(
  "~/workspace/spotmisc/packages/SPOT_2.7.6.tar.gz",
  repos = NULL,
  type = "source"
)
install.packages(
  "~/workspace/spotmisc/packages/SPOTMisc_1.6.6.tar.gz",
  repos = NULL,
  type = "source"
)
library("SPOTMisc")
packageVersion("SPOTMisc")
library("SPOT")
packageVersion("SPOT")

mnist <- getMnistData()

# "dropout" =  x[1]: 0.1--0.5
# "dropoutfact" =  x[2]: 0.85--0.95
# "units" = x[3]: 7-9 (128--512)
# "unitsfact" = x[4]:  0.5--1.5
# "learning_rate" =  x[5]: 1e-5--1e-2
# "epochs" = x[6]: 3-6 (10--50)
# "batchsize" = x[7]: 6-9 (64--512)
# "beta_1" =  x[8]: 0.9--0.999
# "beta_2" =  x[9]: 0.99--0.9999
# "layers" =  x[10]: 1--5
# "epsilon" = x[11]: 1e-8--1e-8
lower <- c(0.1, 0.85, 7, 0.5, 1e-5, 3,  6,  0.9,  0.99,    1, 1e-8)
upper <- c(0.5, 0.95, 9, 1.5, 1e-2, 6, 9, 0.999, 0.9999, 5, 1e-6)
types <-  rep("numeric", length(lower))
types[c(3, 6, 7)] <- "integer"
f2 <- function(x) {
  2 ^ x
}
transformFun <- rep("identity", length(lower))
transformFun[c(3, 6, 7)] <- "f2"
kerasConf <- getKerasConf()
kerasConf$verbose <- 0
kerasConf$resDummy <- TRUE
Ninit <- 2 * length(lower)
Rinit <- 2
replicates <- 2

res <- spot(
  x = NULL,
  fun = funKerasMnist,
  lower = lower,
  upper = upper,
  control = list(
    funEvals = 20 * length(lower),
    multiStart = 2,
    noise = TRUE,
    types = types,
    transformFun = transformFun,
    replicates = replicates,
    designControl = list(replicates = Rinit,
                         size = Ninit),
    model = buildKriging,
    optimizer = optimDE,
    modelControl = list(target = "ei"),
    plots = FALSE,
    progress = TRUE,
    seedFun = 1,
    seedSPOT = 1,
    verbosity=0
  ),
  kConf = kerasConf,
  data = mnist
)
save(res, file = paste0("resKerasMnist11", as.numeric(Sys.time()), ".RData"))
```

