Skip to contents

Run the full Experiment pipeline (fitting, evaluating, and visualizing).

Usage

run_experiment(
  experiment,
  n_reps = 1,
  parallel_strategy = c("reps"),
  future.globals = NULL,
  future.packages = NULL,
  future.seed = TRUE,
  use_cached = FALSE,
  return_all_cached_reps = FALSE,
  save = FALSE,
  checkpoint_n_reps = 0,
  verbose = 1,
  ...
)

Arguments

experiment

An Experiment object.

n_reps

The number of replicates of the Experiment for this run.

parallel_strategy

A vector with some combination of "reps", "dgps", or "methods". Determines how computation will be distributed across available resources. Currently only the default, "reps", is supported.

future.globals

Character vector of names in the global environment to pass to parallel workers. Passed as the argument of the same name to future.apply::future_lapply and related functions. To set for all runs of the experiment, use the same argument during initialization.

future.packages

Character vector of packages required by parallel workers. Passed as the argument of the same name to future.apply::future_lapply and related functions. To set for all runs of the experiment, use the same argument during initialization.

future.seed

Passed as the argument of the same name in future.apply::future_apply.

use_cached

Logical. If TRUE, find and return previously saved results. If cached results cannot be found, continue as if use_cached was FALSE.

return_all_cached_reps

Logical. If FALSE (default), returns only the fit results for the requested n_reps. If TRUE, returns fit results for the requested n_reps plus any additional cached replicates from the (DGP, Method) combinations in the Experiment. Note that even if return_all_cached_reps = TRUE, only the n_reps replicates are used when evaluating and visualizing the Experiment.

save

If TRUE, save outputs to disk.

checkpoint_n_reps

The number of experiment replicates to compute before saving results to disk. If 0 (the default), no checkpoints are saved.

verbose

Level of verbosity. Default is 1, which prints out messages after major checkpoints in the experiment. If 2, prints additional debugging information for warnings and messages from user-defined functions (in addition to error debugging information). If 0, no messages are printed other than user-defined function error debugging information.

...

Not used.

Value

A named list of results from the simulation experiment with the following entries:

fit_results

A tibble containing results from the fit method. In addition to results columns, has columns named '.rep', '.dgp_name', '.method_name', and the vary_across parameter names if applicable.

eval_results

A list of tibbles containing results from the evaluate method, which evaluates each Evaluator in the Experiment. Length of list is equivalent to the number of Evaluators.

viz_results

A list of tibbles containing results from the visualize method, which visualizes each Visualizer in the Experiment. Length of list is equivalent to the number of Visualizers.

Examples

## create toy DGPs, Methods, Evaluators, and Visualizers

# generate data from normal distribution with n samples
normal_dgp <- create_dgp(
  .dgp_fun = function(n) rnorm(n), .name = "Normal DGP", n = 10
)
# generate data from binomial distribution with n samples
bernoulli_dgp <- create_dgp(
  .dgp_fun = function(n) rbinom(n, 1, 0.5), .name = "Bernoulli DGP", n = 10
)

# compute mean of data
mean_method <- create_method(
  .method_fun = function(x) list(mean = mean(x)), .name = "Mean(x)"
)

# evaluate SD of mean(x) across simulation replicates
sd_mean_eval <- create_evaluator(
  .eval_fun = function(fit_results, vary_params = NULL) {
    group_vars <- c(".dgp_name", ".method_name", vary_params)
    fit_results %>%
      dplyr::group_by(dplyr::across(tidyselect::all_of(group_vars))) %>%
      dplyr::summarise(sd = sd(mean), .groups = "keep")
  },
  .name = "SD of Mean(x)"
)

# plot SD of mean(x) across simulation replicates
sd_mean_plot <- create_visualizer(
  .viz_fun = function(fit_results, eval_results, vary_params = NULL,
                      eval_name = "SD of Mean(x)") {
    if (!is.null(vary_params)) {
      add_aes <- ggplot2::aes(
        x = .data[[unique(vary_params)]], y = sd, color = .dgp_name
      )
    } else {
      add_aes <- ggplot2::aes(x = .dgp_name, y = sd)
    }
    plt <- ggplot2::ggplot(eval_results[[eval_name]]) +
      add_aes +
      ggplot2::geom_point()
    if (!is.null(vary_params)) {
      plt <- plt + ggplot2::geom_line()
    }
    return(plt)
  },
  .name = "SD of Mean(x) Plot"
)

# initialize experiment with toy DGPs, Methods, Evaluators, and Visualizers
experiment <- create_experiment(
  name = "Experiment Name",
  dgp_list = list(`Normal DGP` = normal_dgp, `Bernoulli DGP` = bernoulli_dgp),
  method_list = list(`Mean(x)` = mean_method),
  evaluator_list = list(`SD of Mean(x)` = sd_mean_eval),
  visualizer_list = list(`SD of Mean(x) Plot` = sd_mean_plot)
)

# initialize empty experiment with user-defined directory for saving results
experiment <- create_experiment(
  name = "Experiment Name",
  dgp_list = list(`Normal DGP` = normal_dgp, `Bernoulli DGP` = bernoulli_dgp),
  method_list = list(`Mean(x)` = mean_method),
  evaluator_list = list(`SD of Mean(x)` = sd_mean_eval),
  visualizer_list = list(`SD of Mean(x) Plot` = sd_mean_plot),
  save_dir = 'path/to/directory'
)

# initialize experiment with toy DGPs, Methods, Evaluators, and Visualizers
# using piping %>%
experiment <- create_experiment(name = "Experiment Name") %>%
  add_dgp(normal_dgp) %>%
  add_dgp(bernoulli_dgp) %>%
  add_method(mean_method) %>%
  add_evaluator(sd_mean_eval) %>%
  add_visualizer(sd_mean_plot)

# run experiment with 2 replicates
results <- run_experiment(experiment, n_reps = 2)
#> Fitting Experiment Name...
#> 2 reps completed (totals: 2/2) | time taken: 0.044994 minutes
#> ==============================
#> Evaluating Experiment Name...
#> Evaluation completed | time taken: 0.000089 minutes
#> ==============================
#> Visualizing Experiment Name...
#> Visualization completed | time taken: 0.000043 minutes
#> ==============================
# uncomment below to view results
# results

# run experiment with varying number of samples n
experiment <- experiment %>%
  add_vary_across(
    .dgp = c("Normal DGP", "Bernoulli DGP"), n = c(1, 10)
  )
# run vary-across experiment with 2 replicates
results <- run_experiment(experiment, n_reps = 2)
#> Fitting Experiment Name...
#> 2 reps completed (totals: 2/2) | time taken: 0.075092 minutes
#> ==============================
#> Evaluating Experiment Name...
#> Evaluation completed | time taken: 0.000101 minutes
#> ==============================
#> Visualizing Experiment Name...
#> Visualization completed | time taken: 0.000069 minutes
#> ==============================
# uncomment below to view results
# results

# `run_experiment()` above is equivalent to the following sequence of calls
fit_results <- fit_experiment(experiment, n_reps = 2)
#> Fitting Experiment Name...
#> 2 reps completed (totals: 2/2) | time taken: 0.075787 minutes
#> ==============================
eval_results <- evaluate_experiment(experiment, fit_results)
#> Evaluating Experiment Name...
#> Evaluation completed | time taken: 0.000101 minutes
#> ==============================
viz_results <- visualize_experiment(experiment, fit_results, eval_results)
#> Visualizing Experiment Name...
#> Visualization completed | time taken: 0.000070 minutes
#> ==============================

# generate data from all DGPs (and vary across components) in experiment
data_out <- generate_data(experiment, n_reps = 1)