


'''simple docstring'''



import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""", datefmt="""%m/%d/%Y %H:%M:%S""", level=logging.INFO
)
_lowercase      :      str            =					logging.getLogger(__name__)

if __name__ == "__main__":
	_lowercase      :      Optional[Any]            =					argparse.ArgumentParser(
	    description="""Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"""
	)
	parser.add_argument(
	    """--data_file""", type=str, default="""data/dump.bert-base-uncased.pickle""", help="""The binarized dataset."""
	)
	parser.add_argument(
	    """--token_counts_dump""", type=str, default="""data/token_counts.bert-base-uncased.pickle""", help="""The dump file."""
	)
	parser.add_argument("""--vocab_size""", default=30522, type=int)
	_lowercase      :      Optional[int]            =					parser.parse_args()

	logger.info(F"""Loading data from {args.data_file}""")
	with open(args.data_file, """rb""") as fp:
		_lowercase      :      int            =					pickle.load(fp)

	logger.info("""Counting occurrences for MLM.""")
	_lowercase      :      int            =					Counter()
	for tk_ids in data:
		counter.update(tk_ids)
	_lowercase      :      Dict            =					[0] * args.vocab_size
	for k, v in counter.items():
		_lowercase      :      str            =					v

	logger.info(F"""Dump to {args.token_counts_dump}""")
	with open(args.token_counts_dump, """wb""") as handle:
		pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)









'''simple docstring'''



from .data_collator import (
    DataCollatorForLanguageModeling,
    DataCollatorForPermutationLanguageModeling,
    DataCollatorForSeqaSeq,
    DataCollatorForSOP,
    DataCollatorForTokenClassification,
    DataCollatorForWholeWordMask,
    DataCollatorWithPadding,
    DefaultDataCollator,
    default_data_collator,
)
from .metrics import glue_compute_metrics, xnli_compute_metrics
from .processors import (
    DataProcessor,
    InputExample,
    InputFeatures,
    SingleSentenceClassificationProcessor,
    SquadExample,
    SquadFeatures,
    SquadVaProcessor,
    SquadVaProcessor,
    glue_convert_examples_to_features,
    glue_output_modes,
    glue_processors,
    glue_tasks_num_labels,
    squad_convert_examples_to_features,
    xnli_output_modes,
    xnli_processors,
    xnli_tasks_num_labels,
)





