#!/usr/bin/env python3
'''
	# Default experiments

	> This is an *entrypoint file* runnable via ``./experiments/default.py``

	File to run the main/ default experiments.  
	
	=> But it may be also interesting to take a look at this source →
'''
from core.evaluation import Results, Evaluation

from core.corpus import TwentyNews, Wiktionary

from core.model.transformer import IsSCDBert, IsNextSCDBert, SelectSCDBert, GivenTextFindSCDBert
from core.model.scdmatrix import iSCDMatrix, MPSCDMatrix

if __name__ == "__main__":
	e = Evaluation(samples=5, seeds=[123, 456])
	
	e.set_corpus(TwentyNews()) 
	e.set_annotator(Wiktionary) 

	# default models 
	e.add_models([iSCDMatrix, IsSCDBert, IsNextSCDBert, SelectSCDBert, GivenTextFindSCDBert])

	# manual threshold often works better 
	e.add_model(iSCDMatrix, manual_threshold=0.55)

	# works better when not using d2v
	e.add_model(MPSCDMatrix, scd_mapping='ia')

	# with splitted annotator worse performance (task too difficult?)
	e.add_model(MPSCDMatrix, split_annotator=False)
	e.add_model(GivenTextFindSCDBert, split_annotator=False)

	e.run(mail_sample=True, mail_all=False)

	r = Results()
	r.write_json()
	r.write_csv()