#!/usr/bin/env python3

"""
	# Test Hyperparameter

	> This is an *entrypoint file* runnable via ``./experiments/hyperparameter.py``

	Run multiple hyperparameters to figure out best ones.

	=> It may be also interesting to take a look at this source →  
"""

from core.evaluation import Results, Evaluation

from core.corpus import TwentyNews, Wiktionary

from core.model.transformer import IsNextSCDBert

if __name__ == "__main__":
	e = Evaluation(samples=2, seeds=[123, 456])
	
	e.set_corpus(TwentyNews()) 
	e.set_annotator(Wiktionary) 

	# default
	e.add_model(IsNextSCDBert, 
		learning_rate=5e-5, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-8, weight_decay=0.01,
		warmup_steps=500, num_train_epochs=3)

	# no warmup
	e.add_model(IsNextSCDBert, 
		learning_rate=5e-5, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-8, weight_decay=0.01,
		warmup_steps=0, num_train_epochs=3)

	# epochs
	e.add_model(IsNextSCDBert, 
		learning_rate=5e-5, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-8, weight_decay=0.01,
		warmup_steps=500, num_train_epochs=2)

	e.add_model(IsNextSCDBert, 
		learning_rate=5e-5, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-8, weight_decay=0.01,
		warmup_steps=500, num_train_epochs=4)
	
	# weight decay
	e.add_model(IsNextSCDBert, 
		learning_rate=5e-5, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-8, weight_decay=0,
		warmup_steps=500, num_train_epochs=3)

	# learning rate
	e.add_model(IsNextSCDBert, 
		learning_rate=0.001, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-8, weight_decay=0.01,
		warmup_steps=500, num_train_epochs=3)

	# betas
	e.add_model(IsNextSCDBert, 
		learning_rate=5e-5, adam_beta1=0.8, adam_beta2=0.99, adam_epsilon=1e-8, weight_decay=0.01,
		warmup_steps=500, num_train_epochs=3)

	e.run(mail_sample=True, mail_all=False)
	
	r = Results()
	r.write_json()
	r.write_csv()