Chapter 5 Program synthesis experiments
experiment_slug <- "2023-05-08-psynth"
working_directory <- paste0(
"experiments/",
experiment_slug,
"/analysis/"
)
if (exists("bookdown_wd_prefix")) {
working_directory <- paste0(
bookdown_wd_prefix,
working_directory
)
}5.1 Dependencies
library(tidyverse)
library(ggplot2)
library(cowplot)
library(RColorBrewer)
library(khroma)
library(rstatix)
library(knitr)
source("https://gist.githubusercontent.com/benmarwick/2a1bb0133ff568cbe28d/raw/fb53bd97121f7f9ce947837ef1a4c65a73bffb3f/geom_flat_violin.R")print(version)## _
## platform aarch64-apple-darwin20
## arch aarch64
## os darwin20
## system aarch64, darwin20
## status
## major 4
## minor 2.1
## year 2022
## month 06
## day 23
## svn rev 82513
## language R
## version.string R version 4.2.1 (2022-06-23)
## nickname Funny-Looking Kid5.2 Setup
# Configure our default graphing theme
theme_set(theme_cowplot())
# Create a directory to store plots
plot_directory <- paste0(working_directory, "plots/")
dir.create(plot_directory, showWarnings=FALSE)5.2.1 Load summary data
summary_data_loc <- paste0(working_directory, "data/aggregate.csv")
summary_data <- read_csv(summary_data_loc)## Rows: 3120 Columns: 73
## ── Column specification ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
## Delimiter: ","
## chr (11): ANCESTOR_FILE_PATH, EVAL_FIT_EST_MODE, EVAL_MODE, POP_INIT_MODE, P...
## dbl (62): EVAL_CPU_CYCLES_PER_TEST, EVAL_MAX_PHYLO_SEARCH_DEPTH, MAX_ACTIVE_...
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.summary_data <- summary_data %>%
mutate(
eval_mode_row = case_when(
EVAL_MODE == "full" & TEST_DOWNSAMPLE_RATE == "1" ~ "down-sample",
EVAL_MODE == "full" & NUM_COHORTS == "1" ~ "cohort",
.default = EVAL_MODE
),
evals_per_gen = case_when(
EVAL_MODE == "cohort" ~ 1.0 / NUM_COHORTS,
EVAL_MODE == "down-sample" ~ TEST_DOWNSAMPLE_RATE,
EVAL_MODE == "full" ~ 1.0
),
EVAL_FIT_EST_MODE = case_when(
EVAL_FIT_EST_MODE == "ancestor-opt" ~ "ancestor",
EVAL_FIT_EST_MODE == "relative-opt" ~ "relative",
.default = EVAL_FIT_EST_MODE
),
.keep = "all"
) %>%
mutate(
evals_per_gen = as.factor(evals_per_gen),
PROBLEM = as.factor(PROBLEM),
SELECTION = as.factor(SELECTION),
EVAL_MODE = as.factor(EVAL_MODE),
NUM_COHORTS = as.factor(NUM_COHORTS),
TEST_DOWNSAMPLE_RATE = as.factor(TEST_DOWNSAMPLE_RATE),
EVAL_FIT_EST_MODE = factor(
EVAL_FIT_EST_MODE,
levels = c(
"none",
"ancestor",
"relative"
),
labels = c(
"None",
"Ancestor",
"Relative"
)
),
.keep = "all"
)
solution_counts <- summary_data %>%
group_by(
PROBLEM,
evals_per_gen,
eval_mode_row,
EVAL_FIT_EST_MODE,
EVAL_MODE
) %>%
summarize(
solution_count = sum(found_solution == "1"),
replicates = n(),
no_solution_count = n() - sum(found_solution == "1")
)## `summarise()` has grouped output by 'PROBLEM', 'evals_per_gen', 'eval_mode_row',
## 'EVAL_FIT_EST_MODE'. You can override using the `.groups` argument.print(solution_counts, n=140)## # A tibble: 104 × 8
## # Groups: PROBLEM, evals_per_gen, eval_mode_row, EVAL_FIT_EST_MODE [104]
## PROBLEM evals_per_gen eval_…¹ EVAL_…² EVAL_…³ solut…⁴ repli…⁵ no_so…⁶
## <fct> <fct> <chr> <fct> <fct> <int> <int> <int>
## 1 fizz-buzz 0.01 cohort None cohort 0 30 30
## 2 fizz-buzz 0.01 cohort Ancest… cohort 2 30 28
## 3 fizz-buzz 0.01 cohort Relati… cohort 3 30 27
## 4 fizz-buzz 0.01 down-s… None down-s… 0 30 30
## 5 fizz-buzz 0.01 down-s… Ancest… down-s… 0 30 30
## 6 fizz-buzz 0.01 down-s… Relati… down-s… 0 30 30
## 7 fizz-buzz 0.05 cohort None cohort 5 30 25
## 8 fizz-buzz 0.05 cohort Ancest… cohort 3 30 27
## 9 fizz-buzz 0.05 cohort Relati… cohort 7 30 23
## 10 fizz-buzz 0.05 down-s… None down-s… 20 30 10
## 11 fizz-buzz 0.05 down-s… Ancest… down-s… 2 30 28
## 12 fizz-buzz 0.05 down-s… Relati… down-s… 2 30 28
## 13 fizz-buzz 0.1 cohort None cohort 1 30 29
## 14 fizz-buzz 0.1 cohort Ancest… cohort 3 30 27
## 15 fizz-buzz 0.1 cohort Relati… cohort 9 30 21
## 16 fizz-buzz 0.1 down-s… None down-s… 8 30 22
## 17 fizz-buzz 0.1 down-s… Ancest… down-s… 8 30 22
## 18 fizz-buzz 0.1 down-s… Relati… down-s… 7 30 23
## 19 fizz-buzz 0.5 cohort None cohort 0 30 30
## 20 fizz-buzz 0.5 cohort Ancest… cohort 9 30 21
## 21 fizz-buzz 0.5 cohort Relati… cohort 6 30 24
## 22 fizz-buzz 0.5 down-s… None down-s… 0 30 30
## 23 fizz-buzz 0.5 down-s… Ancest… down-s… 7 30 23
## 24 fizz-buzz 0.5 down-s… Relati… down-s… 7 30 23
## 25 fizz-buzz 1 cohort None full 0 30 30
## 26 fizz-buzz 1 down-s… None full 0 30 30
## 27 grade 0.01 cohort None cohort 20 30 10
## 28 grade 0.01 cohort Ancest… cohort 18 30 12
## 29 grade 0.01 cohort Relati… cohort 23 30 7
## 30 grade 0.01 down-s… None down-s… 1 30 29
## 31 grade 0.01 down-s… Ancest… down-s… 10 30 20
## 32 grade 0.01 down-s… Relati… down-s… 11 30 19
## 33 grade 0.05 cohort None cohort 18 30 12
## 34 grade 0.05 cohort Ancest… cohort 13 30 17
## 35 grade 0.05 cohort Relati… cohort 19 30 11
## 36 grade 0.05 down-s… None down-s… 22 30 8
## 37 grade 0.05 down-s… Ancest… down-s… 12 30 18
## 38 grade 0.05 down-s… Relati… down-s… 11 30 19
## 39 grade 0.1 cohort None cohort 12 30 18
## 40 grade 0.1 cohort Ancest… cohort 20 30 10
## 41 grade 0.1 cohort Relati… cohort 15 30 15
## 42 grade 0.1 down-s… None down-s… 22 30 8
## 43 grade 0.1 down-s… Ancest… down-s… 13 30 17
## 44 grade 0.1 down-s… Relati… down-s… 11 30 19
## 45 grade 0.5 cohort None cohort 4 30 26
## 46 grade 0.5 cohort Ancest… cohort 2 30 28
## 47 grade 0.5 cohort Relati… cohort 4 30 26
## 48 grade 0.5 down-s… None down-s… 5 30 25
## 49 grade 0.5 down-s… Ancest… down-s… 9 30 21
## 50 grade 0.5 down-s… Relati… down-s… 4 30 26
## 51 grade 1 cohort None full 1 30 29
## 52 grade 1 down-s… None full 1 30 29
## 53 median 0.01 cohort None cohort 0 30 30
## 54 median 0.01 cohort Ancest… cohort 22 30 8
## 55 median 0.01 cohort Relati… cohort 27 30 3
## 56 median 0.01 down-s… None down-s… 8 30 22
## 57 median 0.01 down-s… Ancest… down-s… 13 30 17
## 58 median 0.01 down-s… Relati… down-s… 14 30 16
## 59 median 0.05 cohort None cohort 12 30 18
## 60 median 0.05 cohort Ancest… cohort 23 30 7
## 61 median 0.05 cohort Relati… cohort 25 30 5
## 62 median 0.05 down-s… None down-s… 4 30 26
## 63 median 0.05 down-s… Ancest… down-s… 19 30 11
## 64 median 0.05 down-s… Relati… down-s… 23 30 7
## 65 median 0.1 cohort None cohort 15 30 15
## 66 median 0.1 cohort Ancest… cohort 26 30 4
## 67 median 0.1 cohort Relati… cohort 24 30 6
## 68 median 0.1 down-s… None down-s… 16 30 14
## 69 median 0.1 down-s… Ancest… down-s… 21 30 9
## 70 median 0.1 down-s… Relati… down-s… 22 30 8
## 71 median 0.5 cohort None cohort 5 30 25
## 72 median 0.5 cohort Ancest… cohort 12 30 18
## 73 median 0.5 cohort Relati… cohort 13 30 17
## 74 median 0.5 down-s… None down-s… 2 30 28
## 75 median 0.5 down-s… Ancest… down-s… 15 30 15
## 76 median 0.5 down-s… Relati… down-s… 13 30 17
## 77 median 1 cohort None full 1 30 29
## 78 median 1 down-s… None full 1 30 29
## 79 small-or-large 0.01 cohort None cohort 0 30 30
## 80 small-or-large 0.01 cohort Ancest… cohort 1 30 29
## 81 small-or-large 0.01 cohort Relati… cohort 0 30 30
## 82 small-or-large 0.01 down-s… None down-s… 0 30 30
## 83 small-or-large 0.01 down-s… Ancest… down-s… 0 30 30
## 84 small-or-large 0.01 down-s… Relati… down-s… 0 30 30
## 85 small-or-large 0.05 cohort None cohort 0 30 30
## 86 small-or-large 0.05 cohort Ancest… cohort 0 30 30
## 87 small-or-large 0.05 cohort Relati… cohort 1 30 29
## 88 small-or-large 0.05 down-s… None down-s… 0 30 30
## 89 small-or-large 0.05 down-s… Ancest… down-s… 0 30 30
## 90 small-or-large 0.05 down-s… Relati… down-s… 0 30 30
## 91 small-or-large 0.1 cohort None cohort 0 30 30
## 92 small-or-large 0.1 cohort Ancest… cohort 0 30 30
## 93 small-or-large 0.1 cohort Relati… cohort 0 30 30
## 94 small-or-large 0.1 down-s… None down-s… 0 30 30
## 95 small-or-large 0.1 down-s… Ancest… down-s… 0 30 30
## 96 small-or-large 0.1 down-s… Relati… down-s… 0 30 30
## 97 small-or-large 0.5 cohort None cohort 0 30 30
## 98 small-or-large 0.5 cohort Ancest… cohort 0 30 30
## 99 small-or-large 0.5 cohort Relati… cohort 0 30 30
## 100 small-or-large 0.5 down-s… None down-s… 0 30 30
## 101 small-or-large 0.5 down-s… Ancest… down-s… 0 30 30
## 102 small-or-large 0.5 down-s… Relati… down-s… 0 30 30
## 103 small-or-large 1 cohort None full 0 30 30
## 104 small-or-large 1 down-s… None full 0 30 30
## # … with abbreviated variable names ¹eval_mode_row, ²EVAL_FIT_EST_MODE,
## # ³EVAL_MODE, ⁴solution_count, ⁵replicates, ⁶no_solution_count5.2.2 Load time series data
Because runs stop when a solution is found, the over time data aren’t completely fair to compare across replicates.
ts_data_loc <- paste0(working_directory, "data/time_series.csv")
ts_data <- read_csv(ts_data_loc)## Rows: 262694 Columns: 24
## ── Column specification ─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
## Delimiter: ","
## chr (6): EVAL_FIT_EST_MODE, EVAL_MODE, PROBLEM, SELECTION, TESTING_SET_PATH...
## dbl (18): EVAL_MAX_PHYLO_SEARCH_DEPTH, NUM_COHORTS, SEED, TEST_DOWNSAMPLE_RA...
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.ts_data <- ts_data %>%
mutate(
eval_mode_row = case_when(
EVAL_MODE == "full" & TEST_DOWNSAMPLE_RATE == "1" ~ "down-sample",
EVAL_MODE == "full" & NUM_COHORTS == "1" ~ "cohort",
.default = EVAL_MODE
),
evals_per_gen = case_when(
EVAL_MODE == "cohort" ~ 1.0 / NUM_COHORTS,
EVAL_MODE == "down-sample" ~ TEST_DOWNSAMPLE_RATE,
EVAL_MODE == "full" ~ 1.0
),
EVAL_FIT_EST_MODE = case_when(
EVAL_FIT_EST_MODE == "ancestor-opt" ~ "ancestor",
EVAL_FIT_EST_MODE == "relative-opt" ~ "relative",
.default = EVAL_FIT_EST_MODE
),
.keep = "all"
) %>%
mutate(
evals_per_gen = as.factor(evals_per_gen),
PROBLEM = as.factor(PROBLEM),
SELECTION = as.factor(SELECTION),
EVAL_MODE = as.factor(EVAL_MODE),
NUM_COHORTS = as.factor(NUM_COHORTS),
TEST_DOWNSAMPLE_RATE = as.factor(TEST_DOWNSAMPLE_RATE),
EVAL_FIT_EST_MODE = factor(
EVAL_FIT_EST_MODE,
levels = c(
"none",
"ancestor",
"relative"
),
labels = c(
"None",
"Ancestor",
"Relative"
)
),
.keep = "all"
)
avg_across_time_data <- ts_data %>%
group_by(
PROBLEM,
evals_per_gen,
eval_mode_row,
EVAL_FIT_EST_MODE,
EVAL_MODE,
SEED
) %>%
summarize(
avg_entropy_selected_ids = mean(entropy_selected_ids),
avg_genotype_pairwise_distance = mean(mean_genotype_pairwise_distance),
avg_num_unique_selected = mean(num_unique_selected),
max_genotype_pairwise_distance = max(mean_genotype_pairwise_distance)
)## `summarise()` has grouped output by 'PROBLEM', 'evals_per_gen', 'eval_mode_row',
## 'EVAL_FIT_EST_MODE', 'EVAL_MODE'. You can override using the `.groups` argument.5.3 Problem-solving success
plt_solutions <- function(data, problem) {
data %>%
filter(PROBLEM == problem) %>%
ggplot(
aes(
x = EVAL_FIT_EST_MODE,
y = solution_count,
fill = EVAL_FIT_EST_MODE
)
) +
geom_col() +
scale_y_continuous(
limits = c(0, 30),
breaks = seq(0, 30),
labels = seq(0, 30)
) +
scale_fill_bright() +
scale_color_bright() +
facet_grid(
eval_mode_row ~ evals_per_gen
) +
labs(title = problem) +
theme(
legend.position = "none",
axis.text.x = element_text(angle = 45, hjust = 1)
)
ggsave(
filename = paste0(plot_directory, problem, "-final.pdf"),
height = 15,
width = 7
)
}
# plt_solutions(solution_counts, "small-or-large")
plt_solutions(solution_counts, "median")
plt_solutions(solution_counts, "grade")
plt_solutions(solution_counts, "fizz-buzz")sol_stats_data <- summary_data %>%
filter(EVAL_MODE != "full" & EVAL_MODE != "cohort-full-compete") %>%
# filter(PROBLEM != "small-or-large") %>%
group_by(
PROBLEM,
evals_per_gen,
EVAL_FIT_EST_MODE,
EVAL_MODE
) %>%
summarize(
solution_count = sum(found_solution == "1"),
replicates = n(),
no_solution_count = n() - sum(found_solution == "1")
) %>%
unite(
"grouping",
PROBLEM,
EVAL_MODE,
evals_per_gen,
sep = "__"
) %>%
select(
!replicates,
) %>%
mutate(
grouping = as.factor(grouping)
)## `summarise()` has grouped output by 'PROBLEM', 'evals_per_gen',
## 'EVAL_FIT_EST_MODE'. You can override using the `.groups` argument.fisher_results <- data.frame(
comparison = character(),
group1 = character(),
group2 = character(),
n = integer(),
p = double(),
p.adj = double(),
p.adj.signif = character()
)
groupings <- levels(sol_stats_data$grouping)
for (g in groupings) {
ft_results <- sol_stats_data %>%
filter(grouping == g) %>%
select(!grouping) %>%
column_to_rownames(var = "EVAL_FIT_EST_MODE") %>%
pairwise_fisher_test(
p.adjust.method = "holm"
) %>%
add_significance("p.adj")
ft_results <- ft_results %>%
mutate(
comparison = rep(g, nrow(ft_results)),
.keep = "all"
) %>%
relocate(comparison)
fisher_results <- rbind(
fisher_results,
ft_results
)
}
kable(fisher_results)| comparison | group1 | group2 | n | p | p.adj | p.adj.signif |
|---|---|---|---|---|---|---|
| fizz-buzz__cohort__0.01 | None | Ancestor | 60 | 4.92e-01 | 9.84e-01 | ns |
| fizz-buzz__cohort__0.01 | None | Relative | 60 | 2.37e-01 | 7.11e-01 | ns |
| fizz-buzz__cohort__0.01 | Ancestor | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| fizz-buzz__cohort__0.05 | None | Ancestor | 60 | 7.06e-01 | 1.00e+00 | ns |
| fizz-buzz__cohort__0.05 | None | Relative | 60 | 7.48e-01 | 1.00e+00 | ns |
| fizz-buzz__cohort__0.05 | Ancestor | Relative | 60 | 2.99e-01 | 8.97e-01 | ns |
| fizz-buzz__cohort__0.1 | None | Ancestor | 60 | 6.12e-01 | 6.12e-01 | ns |
| fizz-buzz__cohort__0.1 | None | Relative | 60 | 1.22e-02 | 3.66e-02 | * |
| fizz-buzz__cohort__0.1 | Ancestor | Relative | 60 | 1.04e-01 | 2.08e-01 | ns |
| fizz-buzz__cohort__0.5 | None | Ancestor | 60 | 1.94e-03 | 5.82e-03 | ** |
| fizz-buzz__cohort__0.5 | None | Relative | 60 | 2.37e-02 | 4.74e-02 | * |
| fizz-buzz__cohort__0.5 | Ancestor | Relative | 60 | 5.52e-01 | 5.52e-01 | ns |
| fizz-buzz__down-sample__0.01 | None | Ancestor | 60 | 1.00e+00 | 1.00e+00 | ns |
| fizz-buzz__down-sample__0.01 | None | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| fizz-buzz__down-sample__0.01 | Ancestor | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| fizz-buzz__down-sample__0.05 | None | Ancestor | 60 | 1.90e-06 | 5.70e-06 | **** |
| fizz-buzz__down-sample__0.05 | None | Relative | 60 | 1.90e-06 | 5.70e-06 | **** |
| fizz-buzz__down-sample__0.05 | Ancestor | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| fizz-buzz__down-sample__0.1 | None | Ancestor | 60 | 1.00e+00 | 1.00e+00 | ns |
| fizz-buzz__down-sample__0.1 | None | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| fizz-buzz__down-sample__0.1 | Ancestor | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| fizz-buzz__down-sample__0.5 | None | Ancestor | 60 | 1.05e-02 | 3.15e-02 | * |
| fizz-buzz__down-sample__0.5 | None | Relative | 60 | 1.05e-02 | 3.15e-02 | * |
| fizz-buzz__down-sample__0.5 | Ancestor | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| grade__cohort__0.01 | None | Ancestor | 60 | 7.89e-01 | 1.00e+00 | ns |
| grade__cohort__0.01 | None | Relative | 60 | 5.67e-01 | 1.00e+00 | ns |
| grade__cohort__0.01 | Ancestor | Relative | 60 | 2.67e-01 | 8.01e-01 | ns |
| grade__cohort__0.05 | None | Ancestor | 60 | 3.01e-01 | 6.02e-01 | ns |
| grade__cohort__0.05 | None | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| grade__cohort__0.05 | Ancestor | Relative | 60 | 1.95e-01 | 5.85e-01 | ns |
| grade__cohort__0.1 | None | Ancestor | 60 | 6.92e-02 | 2.08e-01 | ns |
| grade__cohort__0.1 | None | Relative | 60 | 6.04e-01 | 6.04e-01 | ns |
| grade__cohort__0.1 | Ancestor | Relative | 60 | 2.95e-01 | 5.90e-01 | ns |
| grade__cohort__0.5 | None | Ancestor | 60 | 6.71e-01 | 1.00e+00 | ns |
| grade__cohort__0.5 | None | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| grade__cohort__0.5 | Ancestor | Relative | 60 | 6.71e-01 | 1.00e+00 | ns |
| grade__down-sample__0.01 | None | Ancestor | 60 | 5.58e-03 | 1.12e-02 | * |
| grade__down-sample__0.01 | None | Relative | 60 | 2.47e-03 | 7.41e-03 | ** |
| grade__down-sample__0.01 | Ancestor | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| grade__down-sample__0.05 | None | Ancestor | 60 | 1.82e-02 | 3.64e-02 | * |
| grade__down-sample__0.05 | None | Relative | 60 | 8.87e-03 | 2.66e-02 | * |
| grade__down-sample__0.05 | Ancestor | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| grade__down-sample__0.1 | None | Ancestor | 60 | 3.52e-02 | 7.04e-02 | ns |
| grade__down-sample__0.1 | None | Relative | 60 | 8.87e-03 | 2.66e-02 | * |
| grade__down-sample__0.1 | Ancestor | Relative | 60 | 7.92e-01 | 7.92e-01 | ns |
| grade__down-sample__0.5 | None | Ancestor | 60 | 3.60e-01 | 7.20e-01 | ns |
| grade__down-sample__0.5 | None | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| grade__down-sample__0.5 | Ancestor | Relative | 60 | 2.09e-01 | 6.27e-01 | ns |
| median__cohort__0.01 | None | Ancestor | 60 | 0.00e+00 | 0.00e+00 | **** |
| median__cohort__0.01 | None | Relative | 60 | 0.00e+00 | 0.00e+00 | **** |
| median__cohort__0.01 | Ancestor | Relative | 60 | 1.81e-01 | 1.81e-01 | ns |
| median__cohort__0.05 | None | Ancestor | 60 | 8.21e-03 | 1.64e-02 | * |
| median__cohort__0.05 | None | Relative | 60 | 1.19e-03 | 3.57e-03 | ** |
| median__cohort__0.05 | Ancestor | Relative | 60 | 7.48e-01 | 7.48e-01 | ns |
| median__cohort__0.1 | None | Ancestor | 60 | 4.79e-03 | 1.44e-02 | * |
| median__cohort__0.1 | None | Relative | 60 | 2.92e-02 | 5.84e-02 | ns |
| median__cohort__0.1 | Ancestor | Relative | 60 | 7.31e-01 | 7.31e-01 | ns |
| median__cohort__0.5 | None | Ancestor | 60 | 8.40e-02 | 1.68e-01 | ns |
| median__cohort__0.5 | None | Relative | 60 | 4.70e-02 | 1.41e-01 | ns |
| median__cohort__0.5 | Ancestor | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| median__down-sample__0.01 | None | Ancestor | 60 | 2.79e-01 | 5.58e-01 | ns |
| median__down-sample__0.01 | None | Relative | 60 | 1.80e-01 | 5.40e-01 | ns |
| median__down-sample__0.01 | Ancestor | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| median__down-sample__0.05 | None | Ancestor | 60 | 1.39e-04 | 2.78e-04 | *** |
| median__down-sample__0.05 | None | Relative | 60 | 1.30e-06 | 4.00e-06 | **** |
| median__down-sample__0.05 | Ancestor | Relative | 60 | 3.99e-01 | 3.99e-01 | ns |
| median__down-sample__0.1 | None | Ancestor | 60 | 2.88e-01 | 5.76e-01 | ns |
| median__down-sample__0.1 | None | Relative | 60 | 1.80e-01 | 5.40e-01 | ns |
| median__down-sample__0.1 | Ancestor | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| median__down-sample__0.5 | None | Ancestor | 60 | 3.72e-04 | 1.12e-03 | ** |
| median__down-sample__0.5 | None | Relative | 60 | 2.13e-03 | 4.26e-03 | ** |
| median__down-sample__0.5 | Ancestor | Relative | 60 | 7.96e-01 | 7.96e-01 | ns |
| small-or-large__cohort__0.01 | None | Ancestor | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__cohort__0.01 | None | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__cohort__0.01 | Ancestor | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__cohort__0.05 | None | Ancestor | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__cohort__0.05 | None | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__cohort__0.05 | Ancestor | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__cohort__0.1 | None | Ancestor | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__cohort__0.1 | None | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__cohort__0.1 | Ancestor | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__cohort__0.5 | None | Ancestor | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__cohort__0.5 | None | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__cohort__0.5 | Ancestor | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__down-sample__0.01 | None | Ancestor | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__down-sample__0.01 | None | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__down-sample__0.01 | Ancestor | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__down-sample__0.05 | None | Ancestor | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__down-sample__0.05 | None | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__down-sample__0.05 | Ancestor | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__down-sample__0.1 | None | Ancestor | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__down-sample__0.1 | None | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__down-sample__0.1 | Ancestor | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__down-sample__0.5 | None | Ancestor | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__down-sample__0.5 | None | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
| small-or-large__down-sample__0.5 | Ancestor | Relative | 60 | 1.00e+00 | 1.00e+00 | ns |
# temp <- sol_stats_data %>%
# filter(grouping == "fizz-buzz__cohort__0.5") %>%
# select(!grouping) %>%
# column_to_rownames(var = "EVAL_FIT_EST_MODE")
# pairwise_fisher_test(temp, p.adjust.method = "holm")
# print(
# solution_counts %>% filter(eval_mode_row == "cohort" & PROBLEM == "fizz-buzz"),
# n = 150
# )
# kable(
# fisher_results %>%
# filter(grepl("cohort", comparison) & grepl("fizz-buzz", comparison))
# )