Benchmarking different samplers against a standard ABM

In this tutorial we compare the performance of different sampling methods for the calibration of the paradigmatic model for asset pricing by Brock and Hommes (Journal of Economic Dynamics and Control, 1998).

import numpy as np
from models.economics.brock_hommes import BH4
import matplotlib.pyplot as plt
from black_it.calibrator import Calibrator
import pandas as pd

# generate a single realisation of the BH4 using parameters 
# from the literature (see Platt (2020))

true_params = [0.0, # g1
               0.0, # b1

               0.9, # g2
               0.2, # b2

               0.9, # g3
              -0.2, # b3

               1.01,# g4
               0.01]# b4

parameter_bounds = [[0., 0.,  0.,0., 0.,-1., 1.,0.], # lower bounds
                    [0.1,0.1, 1.,1., 1., 0., 1.1,1.]]  # upper bounds

precisions = [0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01] 

target_series = BH4(true_params, N = 1000, seed = 0)

# plot the target time series

plt.figure(figsize = (7, 5))
plt.plot(target_series)

plt.xlabel('t', fontsize = 14)
plt.ylabel('x', fontsize = 14)
plt.xticks(fontsize=12);
plt.yticks(fontsize=12);

# import a series of samplers to benchmark

from black_it.samplers.halton import HaltonSampler
from black_it.samplers.random_forest import RandomForestSampler
from black_it.samplers.best_batch import BestBatchSampler

all_samplers = [HaltonSampler, RandomForestSampler, BestBatchSampler]

# initialize a set of sampling methods to test,
# note that each method uses the same effective batch size

# 3 single samplers, batch size = 6
batch_size = 6
single_samplers = [[s(batch_size=batch_size)] for s in all_samplers]

# the 3 combinations of 2 different samplers, batch sizes = 3
couple_indices = [[0, 1], [0, 2], [1, 2]]
couple_samplers = []
for ci in couple_indices:
    couple = []
    for i in ci:
        couple.append(all_samplers[i](batch_size=int(batch_size/2)))
    couple_samplers.append(couple)

# a combination of all 3 samplers, batch sizes = 2
triplet_indices = [0, 1, 2]
triplet_samplers = [[all_samplers[i](batch_size=int(batch_size/3)) for i in triplet_indices]]

# define a list with all calibration strategies

all_calibration_strategies = single_samplers + couple_samplers + triplet_samplers

# define a method of moments loss

from black_it.loss_functions.msm import MethodOfMomentsLoss

loss = MethodOfMomentsLoss()

# sample a few random parameters to provide a starting point to the adaptive samplers

from black_it.samplers.random_uniform import RandomUniformSampler

random_sampler = RandomUniformSampler(batch_size)

cal = Calibrator(real_data= target_series, 
             samplers = [random_sampler], 
             loss_function = loss,
             model= BH4, 
             parameters_bounds = parameter_bounds,
             parameters_precision = precisions,
             ensemble_size = 3,
             saving_folder = 'initial_state'
             )

_, _ = cal.calibrate(1)

***
Number of free params:       8.
Explorable param space size: 13988943766831.
***

Selecting 4 processes for the parallel evaluation of the model

BATCH NUMBER:   1
PARAMS SAMPLED: 0

METHOD: RandomUniformSampler
----> sim exec elapsed time: 2.8s
---->   min loss new params: 4.09
---->   avg loss new params: 4.419277934778664e+71
----> avg loss exist params: 4.419277934778664e+71
---->         curr min loss: 4.093299221041861
====>    total elapsed time: 2.8s
Checkpoint saved in 0.0s

# run a series of experiments of 10 epochs each, and save the results

for i, samplers in enumerate(all_calibration_strategies):

    print('Sampling strategy ', i , 'of ', len(all_calibration_strategies))
    sampler_name = ''
    for s in samplers:
        sampler_name+= type(s).__name__

    cal = Calibrator.restore_from_checkpoint('initial_state', model = BH4)

    cal.set_samplers(samplers)

    cal.saving_folder = sampler_name

    cal.calibrate(10)

Sampling strategy  0 of  7

***
Number of free params:       8.
Explorable param space size: 13988943766831.
***

Selecting 4 processes for the parallel evaluation of the model

BATCH NUMBER:   2
PARAMS SAMPLED: 6

METHOD: HaltonSampler
----> sim exec elapsed time: 0.9s
---->   min loss new params: 3.66
---->   avg loss new params: 8.300758412517313e+24
----> avg loss exist params: 2.2096389673893325e+71
---->         curr min loss: 3.6618024049543774
====>    total elapsed time: 0.9s
Checkpoint saved in 0.0s

BATCH NUMBER:   3
PARAMS SAMPLED: 12

METHOD: HaltonSampler
----> sim exec elapsed time: 0.9s
---->   min loss new params: 1.8143899404071114e+25
---->   avg loss new params: 1.9531662506640626e+49
----> avg loss exist params: 1.4730926449262217e+71
---->         curr min loss: 3.6618024049543774
====>    total elapsed time: 0.9s
Checkpoint saved in 0.0s

BATCH NUMBER:   4
PARAMS SAMPLED: 18

METHOD: HaltonSampler
----> sim exec elapsed time: 0.9s
---->   min loss new params: 635.52
---->   avg loss new params: 1.4682776406569115e+65
----> avg loss exist params: 1.1048198507640762e+71
---->         curr min loss: 3.6618024049543774
====>    total elapsed time: 0.9s
Checkpoint saved in 0.0s

BATCH NUMBER:   5
PARAMS SAMPLED: 24

METHOD: HaltonSampler
----> sim exec elapsed time: 0.8s
---->   min loss new params: 1.08
---->   avg loss new params: 1.0259600018245788e+25
----> avg loss exist params: 8.838558806112611e+70
---->         curr min loss: 1.0764783501698882
====>    total elapsed time: 0.8s
Checkpoint saved in 0.0s

BATCH NUMBER:   6
PARAMS SAMPLED: 30

METHOD: HaltonSampler
----> sim exec elapsed time: 0.9s
---->   min loss new params: 1.1663765177260362e+33
---->   avg loss new params: 1.2315345916152899e+57
----> avg loss exist params: 7.365465671760529e+70
---->         curr min loss: 1.0764783501698882
====>    total elapsed time: 0.9s
Checkpoint saved in 0.0s

BATCH NUMBER:   7
PARAMS SAMPLED: 36

METHOD: HaltonSampler
----> sim exec elapsed time: 0.8s
---->   min loss new params: 522.0
---->   avg loss new params: 2.650221569942234e+65
----> avg loss exist params: 6.313260076111268e+70
---->         curr min loss: 1.0764783501698882
====>    total elapsed time: 0.8s
Checkpoint saved in 0.0s

BATCH NUMBER:   8
PARAMS SAMPLED: 42

METHOD: HaltonSampler
----> sim exec elapsed time: 0.9s
---->   min loss new params: 3.13
---->   avg loss new params: 6.323596971065026e+31
----> avg loss exist params: 5.524102566597359e+70
---->         curr min loss: 1.0764783501698882
====>    total elapsed time: 0.9s
Checkpoint saved in 0.0s

BATCH NUMBER:   9
PARAMS SAMPLED: 48

METHOD: HaltonSampler
----> sim exec elapsed time: 0.8s
---->   min loss new params: 9.871204642368983e+32
---->   avg loss new params: 2.0354753999901843e+57
----> avg loss exist params: 4.910313392531009e+70
---->         curr min loss: 1.0764783501698882
====>    total elapsed time: 0.8s
Checkpoint saved in 0.1s

BATCH NUMBER:   10
PARAMS SAMPLED: 54

METHOD: HaltonSampler
----> sim exec elapsed time: 0.9s
---->   min loss new params: 389.87
---->   avg loss new params: 1.9594869275792813e+65
----> avg loss exist params: 4.419284012764835e+70
---->         curr min loss: 1.0764783501698882
====>    total elapsed time: 0.9s
Checkpoint saved in 0.1s

BATCH NUMBER:   11
PARAMS SAMPLED: 60

METHOD: HaltonSampler
----> sim exec elapsed time: 0.7s
---->   min loss new params: 2.95
---->   avg loss new params: 1.9693240640656187e+32
----> avg loss exist params: 4.017530920695304e+70
---->         curr min loss: 1.0764783501698882
====>    total elapsed time: 0.7s
Checkpoint saved in 0.1s
Sampling strategy  1 of  7

***
Number of free params:       8.
Explorable param space size: 13988943766831.
***

Selecting 4 processes for the parallel evaluation of the model

BATCH NUMBER:   2
PARAMS SAMPLED: 6

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.9s
---->   min loss new params: 1.44
---->   avg loss new params: 2.9772293266637943e+72
----> avg loss exist params: 1.7095785600708306e+72
---->         curr min loss: 1.442009376321102
====>    total elapsed time: 2.0s
Checkpoint saved in 0.0s

BATCH NUMBER:   3
PARAMS SAMPLED: 12

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.9s
---->   min loss new params: 3.09
---->   avg loss new params: 8.046150675194563e+17
----> avg loss exist params: 1.1397190400472204e+72
---->         curr min loss: 1.442009376321102
====>    total elapsed time: 2.0s
Checkpoint saved in 0.0s

BATCH NUMBER:   4
PARAMS SAMPLED: 18

METHOD: RandomForestSampler
----> sim exec elapsed time: 1.0s
---->   min loss new params: 158.86
---->   avg loss new params: 1.1850121207855025e+18
----> avg loss exist params: 8.547892800354153e+71
---->         curr min loss: 1.442009376321102
====>    total elapsed time: 1.9s
Checkpoint saved in 0.0s

BATCH NUMBER:   5
PARAMS SAMPLED: 24

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.8s
---->   min loss new params: 230659.17
---->   avg loss new params: 4.2845603081718367e+55
----> avg loss exist params: 6.838314240283323e+71
---->         curr min loss: 1.442009376321102
====>    total elapsed time: 1.9s
Checkpoint saved in 0.0s

BATCH NUMBER:   6
PARAMS SAMPLED: 30

METHOD: RandomForestSampler
----> sim exec elapsed time: 1.1s
---->   min loss new params: 7.16
---->   avg loss new params: 4.952491859756142e+17
----> avg loss exist params: 5.698595200236102e+71
---->         curr min loss: 1.442009376321102
====>    total elapsed time: 2.8s
Checkpoint saved in 0.0s

BATCH NUMBER:   7
PARAMS SAMPLED: 36

METHOD: RandomForestSampler
----> sim exec elapsed time: 1.1s
---->   min loss new params: 4.51
---->   avg loss new params: 28.13
----> avg loss exist params: 4.884510171630944e+71
---->         curr min loss: 1.442009376321102
====>    total elapsed time: 2.7s
Checkpoint saved in 0.1s

BATCH NUMBER:   8
PARAMS SAMPLED: 42

METHOD: RandomForestSampler
----> sim exec elapsed time: 1.1s
---->   min loss new params: 4.59
---->   avg loss new params: 1.4545076704483378e+30
----> avg loss exist params: 4.2739464001770764e+71
---->         curr min loss: 1.442009376321102
====>    total elapsed time: 2.5s
Checkpoint saved in 0.1s

BATCH NUMBER:   9
PARAMS SAMPLED: 48

METHOD: RandomForestSampler
----> sim exec elapsed time: 1.4s
---->   min loss new params: 2.49
---->   avg loss new params: 1.4027803946629642e+22
----> avg loss exist params: 3.799063466824068e+71
---->         curr min loss: 1.442009376321102
====>    total elapsed time: 2.4s
Checkpoint saved in 0.1s

BATCH NUMBER:   10
PARAMS SAMPLED: 54

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.9s
---->   min loss new params: 1.28
---->   avg loss new params: 4.658999191050945e+16
----> avg loss exist params: 3.4191571201416613e+71
---->         curr min loss: 1.2772975970984322
====>    total elapsed time: 2.4s
Checkpoint saved in 0.1s

BATCH NUMBER:   11
PARAMS SAMPLED: 60

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.8s
---->   min loss new params: 1.01
---->   avg loss new params: 1.327724523081595e+40
----> avg loss exist params: 3.1083246546742375e+71
---->         curr min loss: 1.005559758393759
====>    total elapsed time: 1.8s
Checkpoint saved in 0.1s
Sampling strategy  2 of  7

***
Number of free params:       8.
Explorable param space size: 13988943766831.
***

Selecting 4 processes for the parallel evaluation of the model

BATCH NUMBER:   2
PARAMS SAMPLED: 6

METHOD: BestBatchSampler
----> sim exec elapsed time: 1.0s
---->   min loss new params: 1.2743993839456501e+33
---->   avg loss new params: 3.134246498833947e+73
----> avg loss exist params: 1.589219639090867e+73
---->         curr min loss: 4.093299221041861
====>    total elapsed time: 1.0s
Checkpoint saved in 0.0s

BATCH NUMBER:   3
PARAMS SAMPLED: 12

METHOD: BestBatchSampler
----> sim exec elapsed time: 1.0s
---->   min loss new params: 4.69
---->   avg loss new params: 2.3299915681425326e+48
----> avg loss exist params: 1.0594797593939112e+73
---->         curr min loss: 4.093299221041861
====>    total elapsed time: 1.0s
Checkpoint saved in 0.0s

BATCH NUMBER:   4
PARAMS SAMPLED: 18

METHOD: BestBatchSampler
----> sim exec elapsed time: 1.1s
---->   min loss new params: 3.54
---->   avg loss new params: 5.4794875763647744e+72
----> avg loss exist params: 9.315970089545528e+72
---->         curr min loss: 3.537864613089154
====>    total elapsed time: 1.1s
Checkpoint saved in 0.0s

BATCH NUMBER:   5
PARAMS SAMPLED: 24

METHOD: BestBatchSampler
----> sim exec elapsed time: 2.0s
---->   min loss new params: 3.65
---->   avg loss new params: 5.477355418974891e+54
----> avg loss exist params: 7.452776071636422e+72
---->         curr min loss: 3.537864613089154
====>    total elapsed time: 2.0s
Checkpoint saved in 0.1s

BATCH NUMBER:   6
PARAMS SAMPLED: 30

METHOD: BestBatchSampler
----> sim exec elapsed time: 1.7s
---->   min loss new params: 3.06
---->   avg loss new params: 3.648147170157569e+32
----> avg loss exist params: 6.210646726363686e+72
---->         curr min loss: 3.061494620652621
====>    total elapsed time: 1.7s
Checkpoint saved in 0.1s

BATCH NUMBER:   7
PARAMS SAMPLED: 36

METHOD: BestBatchSampler
----> sim exec elapsed time: 1.0s
---->   min loss new params: 2.75
---->   avg loss new params: 3.19
----> avg loss exist params: 5.323411479740301e+72
---->         curr min loss: 2.7469033833359866
====>    total elapsed time: 1.0s
Checkpoint saved in 0.0s

BATCH NUMBER:   8
PARAMS SAMPLED: 42

METHOD: BestBatchSampler
----> sim exec elapsed time: 1.5s
---->   min loss new params: 2.51
---->   avg loss new params: 3.32
----> avg loss exist params: 4.657985044772764e+72
---->         curr min loss: 2.5106289507714665
====>    total elapsed time: 1.5s
Checkpoint saved in 0.1s

BATCH NUMBER:   9
PARAMS SAMPLED: 48

METHOD: BestBatchSampler
----> sim exec elapsed time: 1.4s
---->   min loss new params: 1.69
---->   avg loss new params: 2.8230131715153864e+46
----> avg loss exist params: 4.140431150909123e+72
---->         curr min loss: 1.6890182521981136
====>    total elapsed time: 1.4s
Checkpoint saved in 0.1s

BATCH NUMBER:   10
PARAMS SAMPLED: 54

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.8s
---->   min loss new params: 1.74
---->   avg loss new params: 2.4
----> avg loss exist params: 3.726388035818211e+72
---->         curr min loss: 1.6890182521981136
====>    total elapsed time: 0.9s
Checkpoint saved in 0.1s

BATCH NUMBER:   11
PARAMS SAMPLED: 60

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.8s
---->   min loss new params: 1.71
---->   avg loss new params: 1.371171529418354e+16
----> avg loss exist params: 3.3876254871074644e+72
---->         curr min loss: 1.6890182521981136
====>    total elapsed time: 0.8s
Checkpoint saved in 0.1s
Sampling strategy  3 of  7

***
Number of free params:       8.
Explorable param space size: 13988943766831.
***

Selecting 4 processes for the parallel evaluation of the model

BATCH NUMBER:   2
PARAMS SAMPLED: 6

METHOD: HaltonSampler
----> sim exec elapsed time: 0.4s
---->   min loss new params: 3.66
---->   avg loss new params: 23759.98
----> avg loss exist params: 2.9461852898524434e+71
---->         curr min loss: 3.6618024049543774
====>    total elapsed time: 0.4s

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.5s
---->   min loss new params: 266.38
---->   avg loss new params: 4.929504599938332e+24
----> avg loss exist params: 2.2096389673893325e+71
---->         curr min loss: 3.6618024049543774
====>    total elapsed time: 1.4s
Checkpoint saved in 0.0s

BATCH NUMBER:   3
PARAMS SAMPLED: 12

METHOD: HaltonSampler
----> sim exec elapsed time: 0.4s
---->   min loss new params: 259408.3
---->   avg loss new params: 1.3418466928955868e+25
----> avg loss exist params: 1.7677111739114658e+71
---->         curr min loss: 3.6618024049543774
====>    total elapsed time: 0.4s

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.5s
---->   min loss new params: 5.59
---->   avg loss new params: 9.91
----> avg loss exist params: 1.4730926449262217e+71
---->         curr min loss: 3.6618024049543774
====>    total elapsed time: 1.3s
Checkpoint saved in 0.0s

BATCH NUMBER:   4
PARAMS SAMPLED: 18

METHOD: HaltonSampler
----> sim exec elapsed time: 0.4s
---->   min loss new params: 3.218208797625005e+25
---->   avg loss new params: 4.330494474331893e+33
----> avg loss exist params: 1.2626508385081898e+71
---->         curr min loss: 3.6618024049543774
====>    total elapsed time: 0.4s

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.7s
---->   min loss new params: 4.79
---->   avg loss new params: 4.1039175555312315e+40
----> avg loss exist params: 1.1048194836946663e+71
---->         curr min loss: 3.6618024049543774
====>    total elapsed time: 1.9s
Checkpoint saved in 0.0s

BATCH NUMBER:   5
PARAMS SAMPLED: 24

METHOD: HaltonSampler
----> sim exec elapsed time: 0.4s
---->   min loss new params: 1.0817917199830301e+42
---->   avg loss new params: 3.9053598974827594e+49
----> avg loss exist params: 9.820617632841477e+70
---->         curr min loss: 3.6618024049543774
====>    total elapsed time: 0.4s

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.5s
---->   min loss new params: 3.19
---->   avg loss new params: 1.9816531622380618e+31
----> avg loss exist params: 8.838555869557329e+70
---->         curr min loss: 3.190897002897203
====>    total elapsed time: 1.7s
Checkpoint saved in 0.0s

BATCH NUMBER:   6
PARAMS SAMPLED: 30

METHOD: HaltonSampler
----> sim exec elapsed time: 0.7s
---->   min loss new params: 7.022331859686527e+57
---->   avg loss new params: 3.5277422231985074e+64
----> avg loss exist params: 8.035051111210502e+70
---->         curr min loss: 3.190897002897203
====>    total elapsed time: 0.7s

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.7s
---->   min loss new params: 2.38
---->   avg loss new params: 5475.56
----> avg loss exist params: 7.365463518609625e+70
---->         curr min loss: 2.376892792787833
====>    total elapsed time: 3.5s
Checkpoint saved in 0.0s

BATCH NUMBER:   7
PARAMS SAMPLED: 36

METHOD: HaltonSampler
----> sim exec elapsed time: 0.7s
---->   min loss new params: 606.59
---->   avg loss new params: 3.6625250278652666e+65
----> avg loss exist params: 6.798892219120445e+70
---->         curr min loss: 2.376892792787833
====>    total elapsed time: 0.7s

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.8s
---->   min loss new params: 4.47
---->   avg loss new params: 19.76
----> avg loss exist params: 6.313257060611841e+70
---->         curr min loss: 2.376892792787833
====>    total elapsed time: 2.4s
Checkpoint saved in 0.2s

BATCH NUMBER:   8
PARAMS SAMPLED: 42

METHOD: HaltonSampler
----> sim exec elapsed time: 1.1s
---->   min loss new params: 1.11
---->   avg loss new params: 7.18
----> avg loss exist params: 5.892373256571052e+70
---->         curr min loss: 1.1066869096045393
====>    total elapsed time: 1.1s

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.6s
---->   min loss new params: 6.03
---->   avg loss new params: 27.11
----> avg loss exist params: 5.524099928035361e+70
---->         curr min loss: 1.1066869096045393
====>    total elapsed time: 2.2s
Checkpoint saved in 0.1s

BATCH NUMBER:   9
PARAMS SAMPLED: 48

METHOD: HaltonSampler
----> sim exec elapsed time: 0.6s
---->   min loss new params: 1.14089698130206e+17
---->   avg loss new params: 2.1170674425931233e+25
----> avg loss exist params: 5.199152873445046e+70
---->         curr min loss: 1.1066869096045393
====>    total elapsed time: 0.6s

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.5s
---->   min loss new params: 1.22
---->   avg loss new params: 12.59
----> avg loss exist params: 4.9103110471425434e+70
---->         curr min loss: 1.1066869096045393
====>    total elapsed time: 1.7s
Checkpoint saved in 0.1s

BATCH NUMBER:   10
PARAMS SAMPLED: 54

METHOD: HaltonSampler
----> sim exec elapsed time: 0.6s
---->   min loss new params: 2.785000441888143e+33
---->   avg loss new params: 2.1868878143494303e+41
----> avg loss exist params: 4.651873623608726e+70
---->         curr min loss: 1.1066869096045393
====>    total elapsed time: 0.6s

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.4s
---->   min loss new params: 13.6
---->   avg loss new params: 1.1235228254061286e+32
----> avg loss exist params: 4.419279942428289e+70
---->         curr min loss: 1.1066869096045393
====>    total elapsed time: 1.3s
Checkpoint saved in 0.1s

BATCH NUMBER:   11
PARAMS SAMPLED: 60

METHOD: HaltonSampler
----> sim exec elapsed time: 1.5s
---->   min loss new params: 7.165734019763288e+48
---->   avg loss new params: 2.613140765222106e+57
----> avg loss exist params: 4.2088380404079067e+70
---->         curr min loss: 1.1066869096045393
====>    total elapsed time: 1.5s

METHOD: RandomForestSampler
----> sim exec elapsed time: 1.9s
---->   min loss new params: 6.33
---->   avg loss new params: 9.43
----> avg loss exist params: 4.017527220389366e+70
---->         curr min loss: 1.1066869096045393
====>    total elapsed time: 4.1s
Checkpoint saved in 0.2s
Sampling strategy  4 of  7

***
Number of free params:       8.
Explorable param space size: 13988943766831.
***

Selecting 4 processes for the parallel evaluation of the model

BATCH NUMBER:   2
PARAMS SAMPLED: 6

METHOD: HaltonSampler
----> sim exec elapsed time: 0.5s
---->   min loss new params: 3.66
---->   avg loss new params: 23759.98
----> avg loss exist params: 2.9461852898524434e+71
---->         curr min loss: 3.6618024049543774
====>    total elapsed time: 0.5s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.4s
---->   min loss new params: 3.91
---->   avg loss new params: 9.57
----> avg loss exist params: 2.2096389673893325e+71
---->         curr min loss: 3.6618024049543774
====>    total elapsed time: 0.4s
Checkpoint saved in 0.0s

BATCH NUMBER:   3
PARAMS SAMPLED: 12

METHOD: HaltonSampler
----> sim exec elapsed time: 0.5s
---->   min loss new params: 259408.3
---->   avg loss new params: 1.3418466928955868e+25
----> avg loss exist params: 1.7677111739114658e+71
---->         curr min loss: 3.6618024049543774
====>    total elapsed time: 0.5s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.8s
---->   min loss new params: 3.38
---->   avg loss new params: 3.63
----> avg loss exist params: 1.4730926449262217e+71
---->         curr min loss: 3.3807587026196213
====>    total elapsed time: 0.8s
Checkpoint saved in 0.0s

BATCH NUMBER:   4
PARAMS SAMPLED: 18

METHOD: HaltonSampler
----> sim exec elapsed time: 0.7s
---->   min loss new params: 3.218208797625005e+25
---->   avg loss new params: 4.330494474331893e+33
----> avg loss exist params: 1.2626508385081898e+71
---->         curr min loss: 3.3807587026196213
====>    total elapsed time: 0.7s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.5s
---->   min loss new params: 2.99
---->   avg loss new params: 4.57
----> avg loss exist params: 1.1048194836946663e+71
---->         curr min loss: 2.986363446469386
====>    total elapsed time: 0.5s
Checkpoint saved in 0.0s

BATCH NUMBER:   5
PARAMS SAMPLED: 24

METHOD: HaltonSampler
----> sim exec elapsed time: 0.5s
---->   min loss new params: 1.0817917199830301e+42
---->   avg loss new params: 3.9053598974827594e+49
----> avg loss exist params: 9.820617632841477e+70
---->         curr min loss: 2.986363446469386
====>    total elapsed time: 0.5s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.5s
---->   min loss new params: 2.91
---->   avg loss new params: 4.303297507113467e+39
----> avg loss exist params: 8.838555869557329e+70
---->         curr min loss: 2.9081793471084874
====>    total elapsed time: 0.5s
Checkpoint saved in 0.0s

BATCH NUMBER:   6
PARAMS SAMPLED: 30

METHOD: HaltonSampler
----> sim exec elapsed time: 0.4s
---->   min loss new params: 7.022331859686527e+57
---->   avg loss new params: 3.5277422231985074e+64
----> avg loss exist params: 8.035051111210502e+70
---->         curr min loss: 2.9081793471084874
====>    total elapsed time: 0.4s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.4s
---->   min loss new params: 1.61
---->   avg loss new params: 4.36
----> avg loss exist params: 7.365463518609625e+70
---->         curr min loss: 1.6090030240712685
====>    total elapsed time: 0.4s
Checkpoint saved in 0.1s

BATCH NUMBER:   7
PARAMS SAMPLED: 36

METHOD: HaltonSampler
----> sim exec elapsed time: 0.7s
---->   min loss new params: 606.59
---->   avg loss new params: 3.6625250278652666e+65
----> avg loss exist params: 6.798892219120445e+70
---->         curr min loss: 1.6090030240712685
====>    total elapsed time: 0.7s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.4s
---->   min loss new params: 3.62
---->   avg loss new params: 123998723929308.95
----> avg loss exist params: 6.313257060611841e+70
---->         curr min loss: 1.6090030240712685
====>    total elapsed time: 0.4s
Checkpoint saved in 0.1s

BATCH NUMBER:   8
PARAMS SAMPLED: 42

METHOD: HaltonSampler
----> sim exec elapsed time: 0.5s
---->   min loss new params: 1.11
---->   avg loss new params: 7.18
----> avg loss exist params: 5.892373256571052e+70
---->         curr min loss: 1.1066869096045393
====>    total elapsed time: 0.5s

METHOD: BestBatchSampler
----> sim exec elapsed time: 1.9s
---->   min loss new params: 1.9
---->   avg loss new params: 3.978581939282771e+31
----> avg loss exist params: 5.524099928035361e+70
---->         curr min loss: 1.1066869096045393
====>    total elapsed time: 1.9s
Checkpoint saved in 0.1s

BATCH NUMBER:   9
PARAMS SAMPLED: 48

METHOD: HaltonSampler
----> sim exec elapsed time: 0.7s
---->   min loss new params: 1.14089698130206e+17
---->   avg loss new params: 2.1170674425931233e+25
----> avg loss exist params: 5.199152873445046e+70
---->         curr min loss: 1.1066869096045393
====>    total elapsed time: 0.7s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.8s
---->   min loss new params: 1.13
---->   avg loss new params: 1.48
----> avg loss exist params: 4.9103110471425434e+70
---->         curr min loss: 1.1066869096045393
====>    total elapsed time: 0.8s
Checkpoint saved in 0.1s

BATCH NUMBER:   10
PARAMS SAMPLED: 54

METHOD: HaltonSampler
----> sim exec elapsed time: 0.5s
---->   min loss new params: 2.785000441888143e+33
---->   avg loss new params: 2.1868878143494303e+41
----> avg loss exist params: 4.651873623608726e+70
---->         curr min loss: 1.1066869096045393
====>    total elapsed time: 0.5s

METHOD: BestBatchSampler
----> sim exec elapsed time: 1.1s
---->   min loss new params: 0.94
---->   avg loss new params: 1.14
----> avg loss exist params: 4.419279942428289e+70
---->         curr min loss: 0.942413698087732
====>    total elapsed time: 1.1s
Checkpoint saved in 0.2s

BATCH NUMBER:   11
PARAMS SAMPLED: 60

METHOD: HaltonSampler
----> sim exec elapsed time: 0.9s
---->   min loss new params: 7.165734019763288e+48
---->   avg loss new params: 2.613140765222106e+57
----> avg loss exist params: 4.2088380404079067e+70
---->         curr min loss: 0.942413698087732
====>    total elapsed time: 0.9s

METHOD: BestBatchSampler
----> sim exec elapsed time: 1.0s
---->   min loss new params: 1.18
---->   avg loss new params: 1.470380789543466e+48
----> avg loss exist params: 4.017527220389366e+70
---->         curr min loss: 0.942413698087732
====>    total elapsed time: 1.0s
Checkpoint saved in 0.1s
Sampling strategy  5 of  7

***
Number of free params:       8.
Explorable param space size: 13988943766831.
***

Selecting 4 processes for the parallel evaluation of the model

BATCH NUMBER:   2
PARAMS SAMPLED: 6

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.7s
---->   min loss new params: 12.66
---->   avg loss new params: 30.73
----> avg loss exist params: 2.9461852898524434e+71
---->         curr min loss: 4.093299221041861
====>    total elapsed time: 2.5s

METHOD: BestBatchSampler
----> sim exec elapsed time: 1.4s
---->   min loss new params: 12.8
---->   avg loss new params: 10431.01
----> avg loss exist params: 2.2096389673893325e+71
---->         curr min loss: 4.093299221041861
====>    total elapsed time: 1.4s
Checkpoint saved in 0.0s

BATCH NUMBER:   3
PARAMS SAMPLED: 12

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.6s
---->   min loss new params: 6.81
---->   avg loss new params: 4.3550320248811904e+17
----> avg loss exist params: 1.7677111739114658e+71
---->         curr min loss: 4.093299221041861
====>    total elapsed time: 4.8s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.6s
---->   min loss new params: 8.52
---->   avg loss new params: 9.864850943917757e+62
----> avg loss exist params: 1.4730926465703633e+71
---->         curr min loss: 4.093299221041861
====>    total elapsed time: 0.6s
Checkpoint saved in 0.1s

BATCH NUMBER:   4
PARAMS SAMPLED: 18

METHOD: RandomForestSampler
----> sim exec elapsed time: 2.2s
---->   min loss new params: 9.55
---->   avg loss new params: 3.887305425463723e+32
----> avg loss exist params: 1.2626508399174543e+71
---->         curr min loss: 4.093299221041861
====>    total elapsed time: 3.9s

METHOD: BestBatchSampler
----> sim exec elapsed time: 2.5s
---->   min loss new params: 6.55
---->   avg loss new params: 10.8
----> avg loss exist params: 1.1048194849277725e+71
---->         curr min loss: 4.093299221041861
====>    total elapsed time: 2.6s
Checkpoint saved in 0.2s

BATCH NUMBER:   5
PARAMS SAMPLED: 24

METHOD: RandomForestSampler
----> sim exec elapsed time: 2.4s
---->   min loss new params: 3.03
---->   avg loss new params: 3.9
----> avg loss exist params: 9.820617643802423e+70
---->         curr min loss: 3.0284708010510477
====>    total elapsed time: 4.3s

METHOD: BestBatchSampler
----> sim exec elapsed time: 1.9s
---->   min loss new params: 4.5
---->   avg loss new params: 1.120218120072616e+40
----> avg loss exist params: 8.838555879422181e+70
---->         curr min loss: 3.0284708010510477
====>    total elapsed time: 1.9s
Checkpoint saved in 0.2s

BATCH NUMBER:   6
PARAMS SAMPLED: 30

METHOD: RandomForestSampler
----> sim exec elapsed time: 1.3s
---->   min loss new params: 3.96
---->   avg loss new params: 7.5
----> avg loss exist params: 8.035050799474711e+70
---->         curr min loss: 3.0284708010510477
====>    total elapsed time: 3.5s

METHOD: BestBatchSampler
----> sim exec elapsed time: 1.1s
---->   min loss new params: 4.88
---->   avg loss new params: 7.83
----> avg loss exist params: 7.365463232851817e+70
---->         curr min loss: 3.0284708010510477
====>    total elapsed time: 1.1s
Checkpoint saved in 0.1s

BATCH NUMBER:   7
PARAMS SAMPLED: 36

METHOD: RandomForestSampler
----> sim exec elapsed time: 1.3s
---->   min loss new params: 3.01
---->   avg loss new params: 6.78
----> avg loss exist params: 6.798889138017061e+70
---->         curr min loss: 3.0120008197397503
====>    total elapsed time: 3.3s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.8s
---->   min loss new params: 3.48
---->   avg loss new params: 4.2366396762870067e+18
----> avg loss exist params: 6.313254199587272e+70
---->         curr min loss: 3.0120008197397503
====>    total elapsed time: 0.8s
Checkpoint saved in 0.1s

BATCH NUMBER:   8
PARAMS SAMPLED: 42

METHOD: RandomForestSampler
----> sim exec elapsed time: 1.5s
---->   min loss new params: 2.35
---->   avg loss new params: 5.86
----> avg loss exist params: 5.892370586281454e+70
---->         curr min loss: 2.3490688746866115
====>    total elapsed time: 3.7s

METHOD: BestBatchSampler
----> sim exec elapsed time: 1.1s
---->   min loss new params: 2.1
---->   avg loss new params: 4.5170950241116807e+24
----> avg loss exist params: 5.524097424638862e+70
---->         curr min loss: 2.095820051969746
====>    total elapsed time: 1.2s
Checkpoint saved in 0.1s

BATCH NUMBER:   9
PARAMS SAMPLED: 48

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.6s
---->   min loss new params: 143.9
---->   avg loss new params: 6.3136134841751816e+16
----> avg loss exist params: 5.1991505173071656e+70
---->         curr min loss: 2.095820051969746
====>    total elapsed time: 1.9s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.8s
---->   min loss new params: 2.21
---->   avg loss new params: 2.87
----> avg loss exist params: 4.910308821901212e+70
---->         curr min loss: 2.095820051969746
====>    total elapsed time: 0.8s
Checkpoint saved in 0.1s

BATCH NUMBER:   10
PARAMS SAMPLED: 54

METHOD: RandomForestSampler
----> sim exec elapsed time: 1.1s
---->   min loss new params: 3.78
---->   avg loss new params: 7.144988739148858e+18
----> avg loss exist params: 4.651871515485358e+70
---->         curr min loss: 2.095820051969746
====>    total elapsed time: 2.6s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.6s
---->   min loss new params: 2.16
---->   avg loss new params: 2.52
----> avg loss exist params: 4.4192779397110904e+70
---->         curr min loss: 2.095820051969746
====>    total elapsed time: 0.7s
Checkpoint saved in 0.1s

BATCH NUMBER:   11
PARAMS SAMPLED: 60

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.4s
---->   min loss new params: 2.133175205090705e+26
---->   avg loss new params: 9.445350178094928e+55
----> avg loss exist params: 4.2088361330581827e+70
---->         curr min loss: 2.095820051969746
====>    total elapsed time: 1.6s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.6s
---->   min loss new params: 2.81
---->   avg loss new params: 1.8945860536525166e+39
----> avg loss exist params: 4.0175253997373555e+70
---->         curr min loss: 2.095820051969746
====>    total elapsed time: 0.6s
Checkpoint saved in 0.1s
Sampling strategy  6 of  7

***
Number of free params:       8.
Explorable param space size: 13988943766831.
***

Selecting 4 processes for the parallel evaluation of the model

BATCH NUMBER:   2
PARAMS SAMPLED: 6

METHOD: HaltonSampler
----> sim exec elapsed time: 0.4s
---->   min loss new params: 3.66
---->   avg loss new params: 7.73
----> avg loss exist params: 3.3144584510839986e+71
---->         curr min loss: 3.6618024049543774
====>    total elapsed time: 0.4s

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.4s
---->   min loss new params: 156.11
---->   avg loss new params: 1.421244832993949e+26
----> avg loss exist params: 2.6515667608671987e+71
---->         curr min loss: 3.6618024049543774
====>    total elapsed time: 1.9s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.5s
---->   min loss new params: 4.64
---->   avg loss new params: 9.528436025701058e+22
----> avg loss exist params: 2.2096389673893325e+71
---->         curr min loss: 3.6618024049543774
====>    total elapsed time: 0.5s
Checkpoint saved in 0.0s

BATCH NUMBER:   3
PARAMS SAMPLED: 12

METHOD: HaltonSampler
----> sim exec elapsed time: 0.5s
---->   min loss new params: 94300.57
---->   avg loss new params: 149916.69
----> avg loss exist params: 1.893976257762285e+71
---->         curr min loss: 3.6618024049543774
====>    total elapsed time: 0.5s

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.5s
---->   min loss new params: 59.02
---->   avg loss new params: 589.01
----> avg loss exist params: 1.6572292255419993e+71
---->         curr min loss: 3.6618024049543774
====>    total elapsed time: 2.2s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.6s
---->   min loss new params: 3.49
---->   avg loss new params: 3.74
----> avg loss exist params: 1.4730926449262217e+71
---->         curr min loss: 3.490052814946999
====>    total elapsed time: 0.6s
Checkpoint saved in 0.0s

BATCH NUMBER:   4
PARAMS SAMPLED: 18

METHOD: HaltonSampler
----> sim exec elapsed time: 0.7s
---->   min loss new params: 2.2252400654072278e+17
---->   avg loss new params: 2.0762227039805127e+25
----> avg loss exist params: 1.3257833804335994e+71
---->         curr min loss: 3.490052814946999
====>    total elapsed time: 0.7s

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.6s
---->   min loss new params: 2.25
---->   avg loss new params: 387216.91
----> avg loss exist params: 1.2052576185759994e+71
---->         curr min loss: 2.2472793592156948
====>    total elapsed time: 1.9s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.4s
---->   min loss new params: 2.85
---->   avg loss new params: 5.110258403121564e+31
----> avg loss exist params: 1.1048194836946663e+71
---->         curr min loss: 2.2472793592156948
====>    total elapsed time: 0.4s
Checkpoint saved in 0.1s

BATCH NUMBER:   5
PARAMS SAMPLED: 24

METHOD: HaltonSampler
----> sim exec elapsed time: 0.8s
---->   min loss new params: 2.0543465401664097e+25
---->   avg loss new params: 3.6848605655597034e+33
----> avg loss exist params: 1.0198333695643073e+71
---->         curr min loss: 2.2472793592156948
====>    total elapsed time: 0.8s

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.3s
---->   min loss new params: 3.71
---->   avg loss new params: 4.03
----> avg loss exist params: 9.469881288811425e+70
---->         curr min loss: 2.2472793592156948
====>    total elapsed time: 1.9s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.5s
---->   min loss new params: 2.4
---->   avg loss new params: 2.58
----> avg loss exist params: 8.838555869557329e+70
---->         curr min loss: 2.2472793592156948
====>    total elapsed time: 0.5s
Checkpoint saved in 0.0s

BATCH NUMBER:   6
PARAMS SAMPLED: 30

METHOD: HaltonSampler
----> sim exec elapsed time: 0.8s
---->   min loss new params: 9.534641615636949e+33
---->   avg loss new params: 5.457569779907695e+41
----> avg loss exist params: 8.286146127709996e+70
---->         curr min loss: 2.2472793592156948
====>    total elapsed time: 0.8s

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.4s
---->   min loss new params: 3.61
---->   avg loss new params: 8.29
----> avg loss exist params: 7.798725767256467e+70
---->         curr min loss: 2.2472793592156948
====>    total elapsed time: 1.8s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.4s
---->   min loss new params: 2.21
---->   avg loss new params: 2.49
----> avg loss exist params: 7.365463224631108e+70
---->         curr min loss: 2.2063781203583126
====>    total elapsed time: 0.4s
Checkpoint saved in 0.1s

BATCH NUMBER:   7
PARAMS SAMPLED: 36

METHOD: HaltonSampler
----> sim exec elapsed time: 0.9s
---->   min loss new params: 1.2930856737877442e+42
---->   avg loss new params: 5.852402936836168e+49
----> avg loss exist params: 6.977807265439997e+70
---->         curr min loss: 2.2063781203583126
====>    total elapsed time: 0.9s

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.3s
---->   min loss new params: 6.67
---->   avg loss new params: 137.43
----> avg loss exist params: 6.628916902167997e+70
---->         curr min loss: 2.2063781203583126
====>    total elapsed time: 2.3s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.4s
---->   min loss new params: 1.45
---->   avg loss new params: 1.0122406761007165e+48
----> avg loss exist params: 6.313254192540949e+70
---->         curr min loss: 1.4517270271107627
====>    total elapsed time: 0.4s
Checkpoint saved in 0.0s

BATCH NUMBER:   8
PARAMS SAMPLED: 42

METHOD: HaltonSampler
----> sim exec elapsed time: 0.3s
---->   min loss new params: 4.693912088645729e+57
---->   avg loss new params: 7.321340783291673e+57
----> avg loss exist params: 6.02628809288003e+70
---->         curr min loss: 1.4517270271107627
====>    total elapsed time: 0.3s

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.4s
---->   min loss new params: 8.39
---->   avg loss new params: 34.61
----> avg loss exist params: 5.764275567102637e+70
---->         curr min loss: 1.4517270271107627
====>    total elapsed time: 1.4s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.3s
---->   min loss new params: 3.19
---->   avg loss new params: 1.698842407427921e+47
----> avg loss exist params: 5.524097418473361e+70
---->         curr min loss: 1.4517270271107627
====>    total elapsed time: 0.3s
Checkpoint saved in 0.1s

BATCH NUMBER:   9
PARAMS SAMPLED: 48

METHOD: HaltonSampler
----> sim exec elapsed time: 0.3s
---->   min loss new params: 3.288144112758467e+65
---->   avg loss new params: 7.102222980735781e+65
----> avg loss exist params: 5.303136362623619e+70
---->         curr min loss: 1.4517270271107627
====>    total elapsed time: 0.3s

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.3s
---->   min loss new params: 2.82
---->   avg loss new params: 4.49
----> avg loss exist params: 5.099169579445788e+70
---->         curr min loss: 1.4517270271107627
====>    total elapsed time: 1.5s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.3s
---->   min loss new params: 1.33
---->   avg loss new params: 1.4
----> avg loss exist params: 4.910311446873721e+70
---->         curr min loss: 1.3320791175674513
====>    total elapsed time: 0.3s
Checkpoint saved in 0.1s

BATCH NUMBER:   10
PARAMS SAMPLED: 54

METHOD: HaltonSampler
----> sim exec elapsed time: 0.4s
---->   min loss new params: 601.45
---->   avg loss new params: 1593.66
----> avg loss exist params: 4.734943180913945e+70
---->         curr min loss: 1.3320791175674513
====>    total elapsed time: 0.4s

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.3s
---->   min loss new params: 6.76
---->   avg loss new params: 3.645056588348578e+24
----> avg loss exist params: 4.5716692781238096e+70
---->         curr min loss: 1.3320791175674513
====>    total elapsed time: 1.2s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.3s
---->   min loss new params: 9.80672370968492e+16
---->   avg loss new params: 1.0486335887746384e+32
----> avg loss exist params: 4.419280302186348e+70
---->         curr min loss: 1.3320791175674513
====>    total elapsed time: 0.3s
Checkpoint saved in 0.1s

BATCH NUMBER:   11
PARAMS SAMPLED: 60

METHOD: HaltonSampler
----> sim exec elapsed time: 0.6s
---->   min loss new params: 0.9
---->   avg loss new params: 7.53
----> avg loss exist params: 4.2767228730835636e+70
---->         curr min loss: 0.9031595091997369
====>    total elapsed time: 0.6s

METHOD: RandomForestSampler
----> sim exec elapsed time: 0.5s
---->   min loss new params: 10611.75
---->   avg loss new params: 2.84600288413688e+31
----> avg loss exist params: 4.143075283299702e+70
---->         curr min loss: 0.9031595091997369
====>    total elapsed time: 1.5s

METHOD: BestBatchSampler
----> sim exec elapsed time: 0.4s
---->   min loss new params: 4.82
---->   avg loss new params: 8.096356842026602e+24
----> avg loss exist params: 4.0175275474421353e+70
---->         curr min loss: 0.9031595091997369
====>    total elapsed time: 0.4s
Checkpoint saved in 0.1s

# load the results from the corresponding folders, and plot them!

plt.figure(figsize = (7, 5))
lss = [':']*3 + ['--']*3 + ['-']

for i, sampler in enumerate(all_calibration_strategies):

    # name of the corresponding folder
    sampler_name = ''
    for s in sampler:
        sampler_name += type(s).__name__

    # get array of minimum losses achieved
    losses = pd.read_csv(sampler_name+'/calibration_results.csv')['losses_samp'].cummin()

    sampler_name  = sampler_name.replace("Sampler", "  ")

    # plot the loss curve
    plt.plot(losses, label = sampler_name, ls = lss[i], lw = 2.5)


plt.legend(loc='upper center', bbox_to_anchor=(0.4, -0.14), prop={'size': 14})
plt.xlabel('model calls', fontsize = 14)
plt.ylabel('loss', fontsize = 14);
plt.xticks(fontsize=12);
plt.yticks(fontsize=12);