"""
This file tests the run of experiments at runtime, so simply whether it technically completes the run.
It chooses only 2 runs, with 3 epochs but plots and logs every period. Logging and plotting is not written to disc.
It considers each implemented experiment with each payment rule. Fo each experiment no model_sharing is tested once.
TODO:
- Paul: InputLength should become obsolete Stefan: yes! @assigned Paul
- Paul: Maybe later: Add bne metric to LLLLGG and test for failure
- Stefan: Later: gaussian with fpsb and util_loss
"""
import pytest
# pylint: disable=wrong-import-order
from bnelearn.experiment.configuration_manager import ConfigurationManager
N_RUNS = 2
N_EPOCHS = 3
N_EPOCHS_LLLLGG = 2
# Single item
# id , config, class, known_bne
ids_si, *testdata_si = zip(*[
[
'0 - single_item-symmetric-uniform-fp',
*ConfigurationManager(experiment_type='single_item_uniform_symmetric', n_runs=N_RUNS, n_epochs=N_EPOCHS) \
.get_config(),
True
], [
'1 - single_item-symmetric-uniform-fp-no_model_sharing',
*ConfigurationManager(experiment_type='single_item_uniform_symmetric', n_runs=N_RUNS, n_epochs=N_EPOCHS) \
.set_learning(model_sharing=False)
.get_config(),
True
],
# doesn't test anything significantly different
# ['single_item-symmetric-uniform-vcg', (single_item_uniform_symmetric(2,3, [3], 'second_price')),True],
# too expensive
# ['single_item-symmetric-gaussian-fp', (single_item_gaussian_symmetric(2,3, [4], 'first_price')),True],
[
'2 - single_item-symmetric-gaussian-vcg',
*ConfigurationManager(experiment_type='single_item_gaussian_symmetric', n_runs=N_RUNS, n_epochs=N_EPOCHS) \
.set_setting(payment_rule='second_price', n_players=5)
.get_config(),
True
], [
'3 - single_item-asymmetric-uniform-fp',
*ConfigurationManager(experiment_type='single_item_asymmetric_uniform_overlapping', n_runs=N_RUNS, n_epochs=N_EPOCHS) \
.get_config(),
True
], [
'4 - single_item-asymmetric-uniform-vcg',
*ConfigurationManager(experiment_type='single_item_asymmetric_uniform_overlapping', n_runs=N_RUNS, n_epochs=N_EPOCHS) \
.set_setting(payment_rule='second_price')
.get_config(),
True
], [
'5 - single_item-affiliated-observations',
*ConfigurationManager(experiment_type='affiliated_observations', n_runs=N_RUNS, n_epochs=N_EPOCHS) \
.get_config(),
True
], [
'6 - single_item-mineral-rights',
*ConfigurationManager(experiment_type='mineral_rights', n_runs=N_RUNS, n_epochs=N_EPOCHS) \
.get_config(),
True
]
])
# # Local Global Auctions
# # id , config, class, known_bne
ids_lg, *testdata_lg = zip(*[
[
'0-LLG-fp',
*ConfigurationManager(experiment_type='llg', n_runs=N_RUNS, n_epochs=N_EPOCHS) \
.set_setting(payment_rule='first_price')
.get_config(),
False],
[
'1-LLG-fp-no_model_sharing',
*ConfigurationManager(experiment_type='llg', n_runs=N_RUNS, n_epochs=N_EPOCHS) \
.set_setting(payment_rule='first_price')
.set_learning(model_sharing=False)
.get_config(),
False],
[
'2-LLG-vcg',
*ConfigurationManager(experiment_type='llg', n_runs=N_RUNS, n_epochs=N_EPOCHS) \
.set_setting(payment_rule='vcg')
.get_config(),
True
],
[
'3-LLG-nearest_bid',
*ConfigurationManager(experiment_type='llg', n_runs=N_RUNS, n_epochs=N_EPOCHS)
.set_setting(payment_rule='nearest_bid')
.get_config(),
True
],
[
'4-LLG-nearest_bid_correlated',
*ConfigurationManager(experiment_type='llg', n_runs=N_RUNS, n_epochs=N_EPOCHS)
.set_setting(payment_rule='nearest_bid',
correlation_types='Bernoulli_weights', gamma=0.5)
.get_config(),
True
],
[ # Used to fail when 0.5 didn't, due to neg bids
'5-LLG-nearest_bid_perfectly_correlated',
*ConfigurationManager(experiment_type='llg', n_runs=N_RUNS, n_epochs=N_EPOCHS) \
.set_setting(payment_rule='nearest_bid',
correlation_types='Bernoulli_weights', gamma=1.0)
.get_config(),
True],
[
'6-LLG-full',
*ConfigurationManager(experiment_type='llg_full', n_runs=N_RUNS, n_epochs=N_EPOCHS) \
.get_config(),
True],
[
'7-LLLLGG-fp-no_model_sharing',
*ConfigurationManager(experiment_type='llllgg', n_runs=N_RUNS, n_epochs=N_EPOCHS) \
.set_learning(model_sharing=False)
.get_config(),
False
],
[
'8-LLLLGG-vcg',
*ConfigurationManager(experiment_type='llllgg', n_runs=N_RUNS, n_epochs=N_EPOCHS_LLLLGG) \
.set_setting(payment_rule='vcg')
.get_config(),
False # TODO: vcg BNE not implemented! change this test when done.
]
])
# MultiUnit
# id , config, class, known_bne
ids_mu, *testdata_mu = zip(*[
# TODO: in the following test cases, the "expected bne" has been set to make the tests pass.
# Those where this didn't match Nils's expectations have been marked. @Nils, please take a look.
[
'0-MultiUnit-discr',
*ConfigurationManager(experiment_type='multiunit', n_runs=N_RUNS, n_epochs=N_EPOCHS)
.set_setting(payment_rule='discriminatory')
.get_config(),
True # TODO: Nils said should find False for bne, but finds true
], [
'1-MultiUnit-discr-no_model_sharing',
*ConfigurationManager(experiment_type='multiunit', n_runs=N_RUNS, n_epochs=N_EPOCHS)
.set_setting(payment_rule='discriminatory')
.set_learning(model_sharing=False)
.get_config(),
True # TODO: Nils said should find False for bne, but finds true
], [
'2-MultiUnit-vcg',
*ConfigurationManager(experiment_type='multiunit', n_runs=N_RUNS, n_epochs=N_EPOCHS) \
.set_setting(payment_rule='vcg').get_config(),
True
], [
'3-MultiUnit-uniform',
*ConfigurationManager(experiment_type='multiunit', n_runs=N_RUNS, n_epochs=N_EPOCHS) \
.set_setting(payment_rule='uniform').get_config(),
True
], [
'4-SplitAward-fp',
*ConfigurationManager(experiment_type='splitaward', n_runs=N_RUNS, n_epochs=N_EPOCHS) \
.get_config(),
True # TODO: Nils said should find False for bne, but finds true
], [
'5-SplitAward-fp-no_model_sharing',
*ConfigurationManager(experiment_type='splitaward', n_runs=N_RUNS, n_epochs=N_EPOCHS) \
.set_learning(model_sharing=False)
.get_config(),
True # TODO: Nils said should find False for bne, but finds true
]
])
[docs]def run_auction_test(config, exp_class, known_bne):
config.learning.pretrain_iters = 20
# Log and plot frequent but few
config.logging.enable_logging = False
config.logging.plot_frequency = 1
config.logging.util_loss_frequency = 1
config.logging.plot_points = 10
config.logging.util_loss_batch_size = 2 ** 2
config.logging.util_loss_grid_size = 2 ** 2
config.learning.batch_size = 2 ** 2
config.logging.eval_batch_size = 2 ** 2
# Create and run the experiment
config.hardware.specific_gpu = 0 # TODO: check cuda availability first
experiment = exp_class(config)
assert experiment.known_bne == known_bne, \
"known_bne setting is not as expected!"
success = experiment.run()
assert success, "One or more errors were caught during the experiment runs! (See test logs.)"
[docs]@pytest.mark.parametrize("config, exp_class, known_bne", zip(*testdata_si), ids=ids_si)
def test_single_item_auctions(config, exp_class, known_bne):
run_auction_test(config, exp_class, known_bne)
[docs]@pytest.mark.parametrize("config, exp_class, known_bne", zip(*testdata_lg), ids=ids_lg)
def test_local_global_auctions(config, exp_class, known_bne):
run_auction_test(config, exp_class, known_bne)
[docs]@pytest.mark.parametrize("config, exp_class, known_bne", zip(*testdata_mu), ids=ids_mu)
def test_multi_unit_auctions(config, exp_class, known_bne):
run_auction_test(config, exp_class, known_bne)