






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_nllb import NllbTokenizer
else:
		__magic_name__:					Dict							=     None


__magic_name__:					Optional[Any]							=     logging.get_logger(__name__)


__magic_name__:					List[Any]							=     {"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

__magic_name__:					Optional[Any]							=     {
    "vocab_file": {
        "facebook/nllb-200-distilled-600M": (
            "https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/nllb-200-distilled-600M": (
            "https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json"
        ),
    },
}

__magic_name__:					Dict							=     {
    "facebook/nllb-large-en-ro": 1_024,
    "facebook/nllb-200-distilled-600M": 1_024,
}

# fmt: off
__magic_name__:					List[str]							=     ["ace_Arab", "ace_Latn", "acm_Arab", "acq_Arab", "aeb_Arab", "afr_Latn", "ajp_Arab", "aka_Latn", "amh_Ethi", "apc_Arab", "arb_Arab", "ars_Arab", "ary_Arab", "arz_Arab", "asm_Beng", "ast_Latn", "awa_Deva", "ayr_Latn", "azb_Arab", "azj_Latn", "bak_Cyrl", "bam_Latn", "ban_Latn", "bel_Cyrl", "bem_Latn", "ben_Beng", "bho_Deva", "bjn_Arab", "bjn_Latn", "bod_Tibt", "bos_Latn", "bug_Latn", "bul_Cyrl", "cat_Latn", "ceb_Latn", "ces_Latn", "cjk_Latn", "ckb_Arab", "crh_Latn", "cym_Latn", "dan_Latn", "deu_Latn", "dik_Latn", "dyu_Latn", "dzo_Tibt", "ell_Grek", "eng_Latn", "epo_Latn", "est_Latn", "eus_Latn", "ewe_Latn", "fao_Latn", "pes_Arab", "fij_Latn", "fin_Latn", "fon_Latn", "fra_Latn", "fur_Latn", "fuv_Latn", "gla_Latn", "gle_Latn", "glg_Latn", "grn_Latn", "guj_Gujr", "hat_Latn", "hau_Latn", "heb_Hebr", "hin_Deva", "hne_Deva", "hrv_Latn", "hun_Latn", "hye_Armn", "ibo_Latn", "ilo_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jav_Latn", "jpn_Jpan", "kab_Latn", "kac_Latn", "kam_Latn", "kan_Knda", "kas_Arab", "kas_Deva", "kat_Geor", "knc_Arab", "knc_Latn", "kaz_Cyrl", "kbp_Latn", "kea_Latn", "khm_Khmr", "kik_Latn", "kin_Latn", "kir_Cyrl", "kmb_Latn", "kon_Latn", "kor_Hang", "kmr_Latn", "lao_Laoo", "lvs_Latn", "lij_Latn", "lim_Latn", "lin_Latn", "lit_Latn", "lmo_Latn", "ltg_Latn", "ltz_Latn", "lua_Latn", "lug_Latn", "luo_Latn", "lus_Latn", "mag_Deva", "mai_Deva", "mal_Mlym", "mar_Deva", "min_Latn", "mkd_Cyrl", "plt_Latn", "mlt_Latn", "mni_Beng", "khk_Cyrl", "mos_Latn", "mri_Latn", "zsm_Latn", "mya_Mymr", "nld_Latn", "nno_Latn", "nob_Latn", "npi_Deva", "nso_Latn", "nus_Latn", "nya_Latn", "oci_Latn", "gaz_Latn", "ory_Orya", "pag_Latn", "pan_Guru", "pap_Latn", "pol_Latn", "por_Latn", "prs_Arab", "pbt_Arab", "quy_Latn", "ron_Latn", "run_Latn", "rus_Cyrl", "sag_Latn", "san_Deva", "sat_Beng", "scn_Latn", "shn_Mymr", "sin_Sinh", "slk_Latn", "slv_Latn", "smo_Latn", "sna_Latn", "snd_Arab", "som_Latn", "sot_Latn", "spa_Latn", "als_Latn", "srd_Latn", "srp_Cyrl", "ssw_Latn", "sun_Latn", "swe_Latn", "swh_Latn", "szl_Latn", "tam_Taml", "tat_Cyrl", "tel_Telu", "tgk_Cyrl", "tgl_Latn", "tha_Thai", "tir_Ethi", "taq_Latn", "taq_Tfng", "tpi_Latn", "tsn_Latn", "tso_Latn", "tuk_Latn", "tum_Latn", "tur_Latn", "twi_Latn", "tzm_Tfng", "uig_Arab", "ukr_Cyrl", "umb_Latn", "urd_Arab", "uzn_Latn", "vec_Latn", "vie_Latn", "war_Latn", "wol_Latn", "xho_Latn", "ydd_Hebr", "yor_Latn", "yue_Hant", "zho_Hans", "zho_Hant", "zul_Latn"]





class 	snake_case__ (							_lowerCAmelCase				):
	lowercase__						:					Optional[Any]											=   VOCAB_FILES_NAMES
	lowercase__						:					Dict											=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase__						:					int											=   PRETRAINED_VOCAB_FILES_MAP
	lowercase__						:					str											=   ['''input_ids''', '''attention_mask''']
	lowercase__						:					int											=   NllbTokenizer

	lowercase__						:					List[int]											=   []
	lowercase__						:					List[int]											=   []
	def __init__(    self      ,       lowerCAmelCase__=None      ,       lowerCAmelCase__=None      ,       lowerCAmelCase__="<s>"      ,       lowerCAmelCase__="</s>"      ,       lowerCAmelCase__="</s>"      ,       lowerCAmelCase__="<s>"      ,       lowerCAmelCase__="<unk>"      ,       lowerCAmelCase__="<pad>"      ,       lowerCAmelCase__="<mask>"      ,       lowerCAmelCase__=None      ,       lowerCAmelCase__=None      ,       lowerCAmelCase__=None      ,       lowerCAmelCase__=False      ,       **lowerCAmelCase__      ,       )							->				List[Any]:
							# Mask token behave like a normal word, i.e. include the space before it
							__magic_name__      :       Any       						= AddedToken(lowerCAmelCase__      ,       lstrip=lowerCAmelCase__      ,       rstrip=lowerCAmelCase__			) if isinstance(lowerCAmelCase__      ,       lowerCAmelCase__			) else mask_token
							__magic_name__      :       List[str]       						= legacy_behaviour
							super().__init__(
							    vocab_file=lowerCAmelCase__      ,       tokenizer_file=lowerCAmelCase__      ,       bos_token=lowerCAmelCase__      ,       eos_token=lowerCAmelCase__      ,       sep_token=lowerCAmelCase__      ,       cls_token=lowerCAmelCase__      ,       unk_token=lowerCAmelCase__      ,       pad_token=lowerCAmelCase__      ,       mask_token=lowerCAmelCase__      ,       src_lang=lowerCAmelCase__      ,       tgt_lang=lowerCAmelCase__      ,       additional_special_tokens=lowerCAmelCase__      ,       legacy_behaviour=lowerCAmelCase__      ,       **lowerCAmelCase__      ,       )

							__magic_name__      :       Optional[Any]       						= vocab_file
							__magic_name__      :       int       						= False if not self.vocab_file else True

							__magic_name__      :       Optional[int]       						= FAIRSEQ_LANGUAGE_CODES.copy()

							if additional_special_tokens is not None:
													# Only add those special tokens if they are not already there.
													_additional_special_tokens.extend(
													    [t for t in additional_special_tokens if t not in _additional_special_tokens]			)

							self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}			)
							__magic_name__      :       Optional[int]       						= {
							    lang_code: self.convert_tokens_to_ids(lowerCAmelCase__			) for lang_code in FAIRSEQ_LANGUAGE_CODES
							}

							__magic_name__      :       Dict       						= src_lang if src_lang is not None else """eng_Latn"""
							__magic_name__      :       List[str]       						= self.convert_tokens_to_ids(self._src_lang			)
							__magic_name__      :       int       						= tgt_lang
							self.set_src_lang_special_tokens(self._src_lang			)
	@property
	def       __magic_name__    (    self			)							->				str:
							return self._src_lang
	@src_lang.setter
	def       __magic_name__    (    self      ,       lowerCAmelCase__			)							->				None:
							__magic_name__      :       str       						= new_src_lang
							self.set_src_lang_special_tokens(self._src_lang			)
	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__ = None			)							->				List[int]:
							if token_ids_a is None:
													return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__ = None			)							->				List[int]:
							__magic_name__      :       List[Any]       						= [self.sep_token_id]
							__magic_name__      :       Any       						= [self.cls_token_id]

							if token_ids_a is None:
													return len(cls + token_ids_a + sep			) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]
	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       lowerCAmelCase__      ,       **lowerCAmelCase__			)							->				List[Any]:
							if src_lang is None or tgt_lang is None:
													raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""			)
							__magic_name__      :       List[str]       						= src_lang
							__magic_name__      :       Optional[int]       						= self(lowerCAmelCase__      ,       add_special_tokens=lowerCAmelCase__      ,       return_tensors=lowerCAmelCase__      ,       **lowerCAmelCase__			)
							__magic_name__      :       Optional[Any]       						= self.convert_tokens_to_ids(lowerCAmelCase__			)
							__magic_name__      :       str       						= tgt_lang_id
							return inputs
	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__ = "eng_Latn"      ,       lowerCAmelCase__ = None      ,       lowerCAmelCase__ = "fra_Latn"      ,       **lowerCAmelCase__      ,       )							->				BatchEncoding:
							__magic_name__      :       Any       						= src_lang
							__magic_name__      :       Optional[int]       						= tgt_lang
							return super().prepare_seqaseq_batch(lowerCAmelCase__      ,       lowerCAmelCase__      ,       **lowerCAmelCase__			)
	def       __magic_name__    (    self			)							->				Dict:
							return self.set_src_lang_special_tokens(self.src_lang			)
	def       __magic_name__    (    self			)							->				List[str]:
							return self.set_tgt_lang_special_tokens(self.tgt_lang			)
	def       __magic_name__    (    self      ,       lowerCAmelCase__			)							->				None:
							__magic_name__      :       Optional[int]       						= self.convert_tokens_to_ids(lowerCAmelCase__			)

							if self.legacy_behaviour:
													__magic_name__      :       Tuple       						= []
													__magic_name__      :       str       						= [self.eos_token_id, self.cur_lang_code]
							else:
													__magic_name__      :       str       						= [self.cur_lang_code]
													__magic_name__      :       str       						= [self.eos_token_id]

							__magic_name__      :       Dict       						= self.convert_ids_to_tokens(self.prefix_tokens			)
							__magic_name__      :       List[str]       						= self.convert_ids_to_tokens(self.suffix_tokens			)

							__magic_name__      :       Dict       						= processors.TemplateProcessing(
							    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str      ,       pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str      ,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str      ,       self.prefix_tokens + self.suffix_tokens			)			)      ,       )
	def       __magic_name__    (    self      ,       lowerCAmelCase__			)							->				None:
							__magic_name__      :       List[Any]       						= self.convert_tokens_to_ids(lowerCAmelCase__			)
							if self.legacy_behaviour:
													__magic_name__      :       List[str]       						= []
													__magic_name__      :       Optional[Any]       						= [self.eos_token_id, self.cur_lang_code]
							else:
													__magic_name__      :       List[Any]       						= [self.cur_lang_code]
													__magic_name__      :       Union[str, Any]       						= [self.eos_token_id]

							__magic_name__      :       Dict       						= self.convert_ids_to_tokens(self.prefix_tokens			)
							__magic_name__      :       Union[str, Any]       						= self.convert_ids_to_tokens(self.suffix_tokens			)

							__magic_name__      :       Union[str, Any]       						= processors.TemplateProcessing(
							    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str      ,       pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str      ,       special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str      ,       self.prefix_tokens + self.suffix_tokens			)			)      ,       )







	def       __magic_name__    (    self      ,       lowerCAmelCase__      ,       lowerCAmelCase__ = None			)							->				Tuple[str]:
							if not self.can_save_slow_tokenizer:
													raise ValueError(
													    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
													    """tokenizer."""			)

							if not os.path.isdir(lowerCAmelCase__			):
													logger.error(F'Vocabulary path ({save_directory}) should be a directory.'			)
													return
							__magic_name__      :       int       						= os.path.join(
							    lowerCAmelCase__      ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)

							if os.path.abspath(self.vocab_file			) != os.path.abspath(lowerCAmelCase__			):
													copyfile(self.vocab_file      ,       lowerCAmelCase__			)

							return (out_vocab_file,)








import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
__magic_name__:					str							=     logging.getLogger(__name__)

def    UpperCamelCase						(	):






						"""simple docstring"""







						__magic_name__      :       int       						= argparse.ArgumentParser(
						    description="""Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids)."""			)
						parser.add_argument("""--file_path""",   type=_A,   default="""data/dump.txt""",   help="""The path to the data."""			)
						parser.add_argument("""--tokenizer_type""",   type=_A,   default="""bert""",   choices=["""bert""", """roberta""", """gpt2"""]			)
						parser.add_argument("""--tokenizer_name""",   type=_A,   default="""bert-base-uncased""",   help="""The tokenizer to use."""			)
						parser.add_argument("""--dump_file""",   type=_A,   default="""data/dump""",   help="""The dump file prefix."""			)
						__magic_name__      :       Dict       						= parser.parse_args()

						logger.info(f'Loading Tokenizer ({args.tokenizer_name})'			)
						if args.tokenizer_type == "bert":
												__magic_name__      :       Tuple       						= BertTokenizer.from_pretrained(args.tokenizer_name			)
												__magic_name__      :       List[Any]       						= tokenizer.special_tokens_map["""cls_token"""]  # `[CLS]`
												__magic_name__      :       Optional[int]       						= tokenizer.special_tokens_map["""sep_token"""]  # `[SEP]`
						elif args.tokenizer_type == "roberta":
												__magic_name__      :       Optional[int]       						= RobertaTokenizer.from_pretrained(args.tokenizer_name			)
												__magic_name__      :       List[Any]       						= tokenizer.special_tokens_map["""cls_token"""]  # `<s>`
												__magic_name__      :       Any       						= tokenizer.special_tokens_map["""sep_token"""]  # `</s>`
						elif args.tokenizer_type == "gpt2":
												__magic_name__      :       Any       						= GPTaTokenizer.from_pretrained(args.tokenizer_name			)
												__magic_name__      :       int       						= tokenizer.special_tokens_map["""bos_token"""]  # `<|endoftext|>`
												__magic_name__      :       Optional[int]       						= tokenizer.special_tokens_map["""eos_token"""]  # `<|endoftext|>`

						logger.info(f'Loading text from {args.file_path}'			)
						with open(args.file_path,   """r""",   encoding="""utf8"""			) as fp:
												__magic_name__      :       Tuple       						= fp.readlines()

						logger.info("""Start encoding"""			)
						logger.info(f'{len(_A			)} examples to process.'			)

						__magic_name__      :       List[Any]       						= []
						__magic_name__      :       str       						= 0
						__magic_name__      :       str       						= 10000
						__magic_name__      :       Dict       						= time.time()
						for text in data:
												__magic_name__      :       Tuple       						= f'{bos} {text.strip()} {sep}'
												__magic_name__      :       Optional[int]       						= tokenizer.encode(_A,   add_special_tokens=_A			)
												rslt.append(_A			)

												iter += 1
												if iter % interval == 0:
																		__magic_name__      :       Union[str, Any]       						= time.time()
																		logger.info(f'{iter} examples processed. - {(end-start):.2f}s/{interval}expl'			)
																		__magic_name__      :       Any       						= time.time()
						logger.info("""Finished binarization"""			)
						logger.info(f'{len(_A			)} examples processed.'			)

						__magic_name__      :       Tuple       						= f'{args.dump_file}.{args.tokenizer_name}.pickle'
						__magic_name__      :       Tuple       						= tokenizer.vocab_size
						if vocab_size < (1 << 16):
												__magic_name__      :       Optional[int]       						= [np.uintaa(_A			) for d in rslt]
						else:
												__magic_name__      :       str       						= [np.intaa(_A			) for d in rslt]
						random.shuffle(rslt_			)
						logger.info(f'Dump to {dp_file}'			)
						with open(_A,   """wb"""			) as handle:
												pickle.dump(rslt_,   _A,   protocol=pickle.HIGHEST_PROTOCOL			)


if __name__ == "__main__":
		main()
