## ---- echo=FALSE-------------------------------------------------------------- suppressPackageStartupMessages(library(smoof)) suppressPackageStartupMessages(library(CAISEr)) ## ---- eval=FALSE-------------------------------------------------------------- # # Install if needed # # devtools::install_github("fcampelo/MOEADr") # # suppressPackageStartupMessages(library(smoof)) # suppressPackageStartupMessages(library(MOEADr)) # suppressPackageStartupMessages(library(CAISEr)) # # ### Build function names (instances: UF1 - UF7, dimensions 10 - 40) # fname <- paste0("UF_", 1:7) # dims <- c(10:40) # allfuns <- expand.grid(fname, dims, stringsAsFactors = FALSE) # # # Assemble instances list # instances <- vector(nrow(allfuns), mode = "list") # for (i in 1:length(instances)){ # instances[[i]]$FUN <- paste0(allfuns[i,1], "_", allfuns[i,2]) # } # # ### Build the functions listed above (so that they can be properly used) # for (i in 1:nrow(allfuns)){ # assign(x = instances[[i]]$FUN, # value = MOEADr::make_vectorized_smoof(prob.name = "UF", # dimensions = allfuns[i, 2], # id = as.numeric(strsplit(allfuns[i, 1], "_")[[1]][2]))) # } ## ---- eval=FALSE-------------------------------------------------------------- # # Prepare algorithm function to be used in run_experiment(): # myalgo <- function(type, instance){ # # Input parameters: # # - type (variant to use: "original", "original2", "moead.de" or "moead.de2") # # - instance (instance to be solved, e.g., instance = instances[[i]]) # # All other parameters are set internally # # ## Extract instance information to build the MOEADr problem format # fdef <- unlist(strsplit(instance$FUN, split = "_")) # uffun <- smoof::makeUFFunction(dimensions = as.numeric(fdef[3]), # id = as.numeric(fdef[2])) # fattr <- attr(uffun, "par.set") # prob.dim <- fattr$pars$x$len # # ## Build MOEADr problem list # problem <- list(name = instance$FUN, # xmin = fattr$pars$x$lower, # xmax = fattr$pars$x$upper, # m = attr(uffun, "n.objectives")) # # ## Load presets for the algorithm provided in input 'type' and # ## modify whatever is needed for this particular experiment # de2 <- FALSE # if (type == "moead.de2"){ # de2 <- TRUE # type <- "moead.de" # } # algo.preset <- MOEADr::preset_moead(type) # algo.preset$decomp$H <- 99 # <-- set population size # algo.preset$stopcrit[[1]]$name <- "maxeval" # <-- type of stop criterion # algo.preset$stopcrit[[1]]$maxeval <- 2000 * prob.dim # stop crit. # poly.ind <- which(sapply(algo.preset$variation, # function(x){x$name == "polymut"})) # algo.preset$variation[[poly.ind]]$pm <- 1 / prob.dim # <--- pm = 1/d # if (de2){ # algo.preset$aggfun$name <- "pbi" # algo.preset$aggfun$theta <- 5 # algo.preset$neighbors$name = "x" # } # # ## Run algorithm on "instance" # out <- MOEADr::moead(preset = algo.preset, problem = problem, # showpars = list(show.iters = "none")) # # ## Read reference data to calculate the IGD # Yref <- as.matrix(read.table(paste0("./inst/extdata/pf_data/", # fdef[1], fdef[2], ".dat"))) # IGD = MOEADr::calcIGD(Y = out$Y, Yref = Yref) # # ## Return IGD as field "value" in the output list # return(list(value = IGD)) # } ## ---- eval=FALSE-------------------------------------------------------------- # # Assemble Algorithm.list. Notice that we need to provide an alias for each # # method, since both algorithms have the same '$FUN' argument. # algorithms <- list(list(FUN = "myalgo", # alias = "Original 1", # type = "original"), # list(FUN = "myalgo", # alias = "Original 2", # type = "original2"), # list(FUN = "myalgo", # alias = "MOEAD-DE", # type = "moead.de"), # list(FUN = "myalgo", # alias = "MOEAD-DE2", # type = "moead.de2")) ## ---- eval=FALSE-------------------------------------------------------------- # my.results <- run_experiment(instances = instances, # algorithms = algorithms, # power = 0.8, # Desired power: 80% # power.target = "mean", # on average, # d = 0.5, # to detect differences greater # # than 0.5 standard deviations # sig.level = 0.05, # at a 95% confidence level. # se.max = 0.05, # Measurement error: 5% # dif = "perc", # on the paired percent # # differences of means, # method = "param", # calculated using parametric # # formula. # comparisons = "all.vs.all", # Compare all algorithms # # vs all others, # nstart = 15, # start with 15 runs/algo/inst # nmax = 200, # and do no more than 200 runs/inst. # seed = 1234, # PRNG seed (for reproducibility) # # # # NOTICE: Using all but 1 cores. Change if needed # ncpus = parallel::detectCores() - 1) ## ---- echo=FALSE-------------------------------------------------------------- load("../inst/extdata/vignette_results.RData") ## ---- fig.align="center", fig.width=8, fig.height=8--------------------------- plot(my.results) ## ---- fig.align="center", fig.width=6, fig.height=10-------------------------- suppressPackageStartupMessages(library(car)) algopairs <- paste(my.results$data.summary$Alg1, my.results$data.summary$Alg2, sep = " - ") par(mfrow = c(3, 2)) for (i in seq_along(unique(algopairs))){ tmp <- my.results$data.summary[algopairs == unique(algopairs)[i], ] car::qqPlot(tmp$Phi, pch = 16, las = 1, main = unique(algopairs)[i], ylab = "observed", xlab = "theoretical quantiles") } par(mfrow = c(1, 1)) ## ---- fig.align="center", fig.width=6, fig.height=8--------------------------- par(mfrow = c(3, 2)) for (i in seq_along(unique(algopairs))){ tmp <- my.results$data.summary[algopairs == unique(algopairs)[i], ] boot.means <- CAISEr::boot_sdm(tmp$Phi, boot.R = 999) hist(boot.means, breaks = 30, main = unique(algopairs)[i], las = 1) } par(mfrow = c(1, 1)) ## ---- fig.align="center", fig.width=6, fig.height=4--------------------------- df <- cbind(Comparison = algopairs, my.results$data.summary) suppressPackageStartupMessages(library(ggplot2)) mp <- ggplot(df, aes(x = Comparison, y = Phi, fill = Comparison)) mp + geom_violin(alpha = 0.6, show.legend = FALSE, scale = "width") + geom_boxplot(alpha = 0, show.legend = FALSE, outlier.shape = NA, width = .15) + geom_point(shape = 16, col = "black", fill = "black", alpha = 0.6, position = position_jitter(width = .15)) + geom_abline(slope = 0, intercept = 0, col = "red", lty = 2) + ylab("Percent difference in IGD") + xlab("") + theme(axis.text.x = element_text(angle = 45, hjust = 1)) ## ---- fig.align="center", fig.width=6, fig.height=8--------------------------- ggplot(df, aes(x = Instance, y = Phi, colour = Comparison, ymin = Phi - SE, ymax = Phi + SE)) + geom_pointrange(show.legend = FALSE) + geom_abline(slope = 0, intercept = 0, col = 1, lty = 2) + facet_grid(Comparison ~ .) + theme(axis.text.x = element_text(angle = 60, hjust = 1)) + xlab("") ## ---- fig.align="center", fig.width=6, fig.height=8--------------------------- summary(my.results, test = "wilcoxon")